Use abstract filesystem for stores

app/api/api.go

@@ -34,7 +34,7 @@ import (
 	"github.com/datarhei/core/v16/restream"
 	restreamapp "github.com/datarhei/core/v16/restream/app"
 	"github.com/datarhei/core/v16/restream/replace"
-	"github.com/datarhei/core/v16/restream/store"
+	restreamstore "github.com/datarhei/core/v16/restream/store"
 	"github.com/datarhei/core/v16/rtmp"
 	"github.com/datarhei/core/v16/service"
 	"github.com/datarhei/core/v16/session"
@@ -153,7 +153,8 @@ func (a *api) Reload() error {
 
 	logger := log.New("Core").WithOutput(log.NewConsoleWriter(a.log.writer, log.Lwarn, true))
 
-	store, err := configstore.NewJSON(a.config.path, func() {
+	rootfs, _ := fs.NewDiskFilesystem(fs.DiskConfig{})
+	store, err := configstore.NewJSON(rootfs, a.config.path, func() {
 		a.errorChan <- ErrConfigReload
 	})
 	if err != nil {
@@ -295,7 +296,13 @@ func (a *api) start() error {
 		}
 
 		if cfg.Sessions.Persist {
-			sessionConfig.PersistDir = filepath.Join(cfg.DB.Dir, "sessions")
+			fs, err := fs.NewRootedDiskFilesystem(fs.RootedDiskConfig{
+				Root: filepath.Join(cfg.DB.Dir, "sessions"),
+			})
+			if err != nil {
+				return fmt.Errorf("unable to create filesystem for persisting sessions: %w", err)
+			}
+			sessionConfig.PersistFS = fs
 		}
 
 		sessions, err := session.New(sessionConfig)
@@ -374,11 +381,9 @@ func (a *api) start() error {
 		a.sessions = sessions
 	}
 
-	diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{
-		Name:   "disk",
-		Dir:    cfg.Storage.Disk.Dir,
-		Size:   cfg.Storage.Disk.Size * 1024 * 1024,
-		Logger: a.log.logger.core.WithComponent("FS"),
+	diskfs, err := fs.NewRootedDiskFilesystem(fs.RootedDiskConfig{
+		Root:   cfg.Storage.Disk.Dir,
+		Logger: a.log.logger.core.WithComponent("DiskFS"),
 	})
 	if err != nil {
 		return fmt.Errorf("disk filesystem: %w", err)
@@ -403,21 +408,27 @@ func (a *api) start() error {
 	}
 
 	if a.memfs == nil {
-		memfs := fs.NewMemFilesystem(fs.MemConfig{
-			Name:   "mem",
-			Base:   baseMemFS.String(),
-			Size:   cfg.Storage.Memory.Size * 1024 * 1024,
-			Purge:  cfg.Storage.Memory.Purge,
-			Logger: a.log.logger.core.WithComponent("FS"),
+		memfs, _ := fs.NewMemFilesystem(fs.MemConfig{
+			Logger: a.log.logger.core.WithComponent("MemFS"),
 		})
 
-		a.memfs = memfs
+		memfs.SetMetadata("base", baseMemFS.String())
+
+		sizedfs, _ := fs.NewSizedFilesystem(memfs, cfg.Storage.Memory.Size*1024*1024, cfg.Storage.Memory.Purge)
+
+		a.memfs = sizedfs
 	} else {
-		a.memfs.Rebase(baseMemFS.String())
-		a.memfs.Resize(cfg.Storage.Memory.Size * 1024 * 1024)
+		a.memfs.SetMetadata("base", baseMemFS.String())
+		if sizedfs, ok := a.memfs.(fs.SizedFilesystem); ok {
+			sizedfs.Resize(cfg.Storage.Memory.Size * 1024 * 1024)
+		}
 	}
 
 	for _, s3 := range cfg.Storage.S3 {
+		if _, ok := a.s3fs[s3.Name]; ok {
+			return fmt.Errorf("the name '%s' for a s3 filesystem is already in use", s3.Name)
+		}
+
 		baseS3FS := url.URL{
 			Scheme: "http",
 			Path:   s3.Mountpoint,
@@ -436,7 +447,6 @@ func (a *api) start() error {
 
 		s3fs, err := fs.NewS3Filesystem(fs.S3Config{
 			Name:            s3.Name,
-			Base:            baseS3FS.String(),
 			Endpoint:        s3.Endpoint,
 			AccessKeyID:     s3.AccessKeyID,
 			SecretAccessKey: s3.SecretAccessKey,
@@ -449,9 +459,7 @@ func (a *api) start() error {
 			return fmt.Errorf("s3 filesystem (%s): %w", s3.Name, err)
 		}
 
-		if _, ok := a.s3fs[s3.Name]; ok {
-			return fmt.Errorf("the name '%s' for a filesystem is already in use", s3.Name)
-		}
+		s3fs.SetMetadata("base", baseS3FS.String())
 
 		a.s3fs[s3.Name] = s3fs
 	}
@@ -495,23 +503,23 @@ func (a *api) start() error {
 
 	{
 		a.replacer.RegisterTemplateFunc("diskfs", func(config *restreamapp.Config, section string) string {
-			return a.diskfs.Base()
+			return a.diskfs.Metadata("base")
 		}, nil)
 
 		a.replacer.RegisterTemplateFunc("fs:disk", func(config *restreamapp.Config, section string) string {
-			return a.diskfs.Base()
+			return a.diskfs.Metadata("base")
 		}, nil)
 
 		a.replacer.RegisterTemplateFunc("memfs", func(config *restreamapp.Config, section string) string {
-			return a.memfs.Base()
+			return a.memfs.Metadata("base")
 		}, nil)
 
 		a.replacer.RegisterTemplateFunc("fs:mem", func(config *restreamapp.Config, section string) string {
-			return a.memfs.Base()
+			return a.memfs.Metadata("base")
 		}, nil)
 
 		for name, s3 := range a.s3fs {
-			a.replacer.RegisterTemplate("fs:"+name, s3.Base(), nil)
+			a.replacer.RegisterTemplate("fs:"+name, s3.Metadata("base"), nil)
 		}
 
 		a.replacer.RegisterTemplateFunc("rtmp", func(config *restreamapp.Config, section string) string {
@@ -567,11 +575,24 @@ func (a *api) start() error {
 		filesystems = append(filesystems, fs)
 	}
 
-	store := store.NewJSONStore(store.JSONConfig{
-		Filepath:  cfg.DB.Dir + "/db.json",
-		FFVersion: a.ffmpeg.Skills().FFmpeg.Version,
-		Logger:    a.log.logger.core.WithComponent("ProcessStore"),
-	})
+	var store restreamstore.Store = nil
+
+	{
+		fs, err := fs.NewRootedDiskFilesystem(fs.RootedDiskConfig{
+			Root: cfg.DB.Dir,
+		})
+		if err != nil {
+			return err
+		}
+		store, err = restreamstore.NewJSON(restreamstore.JSONConfig{
+			Filesystem: fs,
+			Filepath:   "/db.json",
+			Logger:     a.log.logger.core.WithComponent("ProcessStore"),
+		})
+		if err != nil {
+			return err
+		}
+	}
 
 	restream, err := restream.New(restream.Config{
 		ID:           cfg.ID,
@@ -645,8 +666,8 @@ func (a *api) start() error {
 	metrics.Register(monitor.NewCPUCollector())
 	metrics.Register(monitor.NewMemCollector())
 	metrics.Register(monitor.NewNetCollector())
-	metrics.Register(monitor.NewDiskCollector(a.diskfs.Base()))
-	metrics.Register(monitor.NewFilesystemCollector("diskfs", diskfs))
+	metrics.Register(monitor.NewDiskCollector(a.diskfs.Metadata("base")))
+	metrics.Register(monitor.NewFilesystemCollector("diskfs", a.diskfs))
 	metrics.Register(monitor.NewFilesystemCollector("memfs", a.memfs))
 	for name, fs := range a.s3fs {
 		metrics.Register(monitor.NewFilesystemCollector(name, fs))
@@ -1395,7 +1416,7 @@ func (a *api) Destroy() {
 
 	// Free the MemFS
 	if a.memfs != nil {
-		a.memfs.DeleteAll()
+		a.memfs.RemoveAll()
 		a.memfs = nil
 	}
 }

app/ffmigrate/main.go

@@ -9,6 +9,7 @@ import (
 	cfgvars "github.com/datarhei/core/v16/config/vars"
 	"github.com/datarhei/core/v16/ffmpeg"
 	"github.com/datarhei/core/v16/io/file"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/log"
 	"github.com/datarhei/core/v16/restream/store"
 
@@ -24,7 +25,9 @@ func main() {
 
 	configfile := cfgstore.Location(os.Getenv("CORE_CONFIGFILE"))
 
-	configstore, err := cfgstore.NewJSON(configfile, nil)
+	diskfs, _ := fs.NewDiskFilesystem(fs.DiskConfig{})
+
+	configstore, err := cfgstore.NewJSON(diskfs, configfile, nil)
 	if err != nil {
 		logger.Error().WithError(err).Log("Loading configuration failed")
 		os.Exit(1)
@@ -117,9 +120,12 @@ func doMigration(logger log.Logger, configstore cfgstore.Store) error {
 	logger.Info().WithField("backup", backupFilepath).Log("Backup created")
 
 	// Load the existing DB
-	datastore := store.NewJSONStore(store.JSONConfig{
+	datastore, err := store.NewJSON(store.JSONConfig{
 		Filepath: cfg.DB.Dir + "/db.json",
 	})
+	if err != nil {
+		return err
+	}
 
 	data, err := datastore.Load()
 	if err != nil {

app/import/import.go

@@ -17,6 +17,7 @@ import (
 	"github.com/datarhei/core/v16/encoding/json"
 	"github.com/datarhei/core/v16/ffmpeg"
 	"github.com/datarhei/core/v16/ffmpeg/skills"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/restream"
 	"github.com/datarhei/core/v16/restream/app"
 	"github.com/datarhei/core/v16/restream/store"
@@ -495,14 +496,14 @@ type importConfigAudio struct {
 	sampling string
 }
 
-func importV1(path string, cfg importConfig) (store.StoreData, error) {
+func importV1(fs fs.Filesystem, path string, cfg importConfig) (store.StoreData, error) {
 	if len(cfg.id) == 0 {
 		cfg.id = uuid.New().String()
 	}
 
 	r := store.NewStoreData()
 
-	jsondata, err := os.ReadFile(path)
+	jsondata, err := fs.ReadFile(path)
 	if err != nil {
 		return r, fmt.Errorf("failed to read data from %s: %w", path, err)
 	}
@@ -1417,9 +1418,19 @@ func probeInput(binary string, config app.Config) app.Probe {
 		return app.Probe{}
 	}
 
+	dummyfs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+	store, err := store.NewJSON(store.JSONConfig{
+		Filesystem: dummyfs,
+		Filepath:   "/",
+		Logger:     nil,
+	})
+	if err != nil {
+		return app.Probe{}
+	}
+
 	rs, err := restream.New(restream.Config{
 		FFmpeg: ffmpeg,
-		Store:  store.NewDummyStore(store.DummyConfig{}),
+		Store:  store,
 	})
 	if err != nil {
 		return app.Probe{}

app/import/import_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/datarhei/core/v16/encoding/json"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/restream/store"
 
 	"github.com/stretchr/testify/require"
@@ -36,8 +37,13 @@ import (
 var id string = "4186b095-7f0a-4e94-8c3d-f17459ab252f"
 
 func testV1Import(t *testing.T, v1Fixture, v4Fixture string, config importConfig) {
+	diskfs, err := fs.NewRootedDiskFilesystem(fs.RootedDiskConfig{
+		Root: ".",
+	})
+	require.NoError(t, err)
+
 	// Import v1 database
-	v4, err := importV1(v1Fixture, config)
+	v4, err := importV1(diskfs, v1Fixture, config)
 	require.Equal(t, nil, err)
 
 	// Reset variants
@@ -50,7 +56,7 @@ func testV1Import(t *testing.T, v1Fixture, v4Fixture string, config importConfig
 	require.Equal(t, nil, err)
 
 	// Read the wanted result
-	wantdatav4, err := os.ReadFile(v4Fixture)
+	wantdatav4, err := diskfs.ReadFile(v4Fixture)
 	require.Equal(t, nil, err)
 
 	var wantv4 store.StoreData

app/import/main.go

@@ -6,6 +6,7 @@ import (
 
 	cfgstore "github.com/datarhei/core/v16/config/store"
 	cfgvars "github.com/datarhei/core/v16/config/vars"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/log"
 	"github.com/datarhei/core/v16/restream/store"
 
@@ -17,18 +18,24 @@ func main() {
 
 	configfile := cfgstore.Location(os.Getenv("CORE_CONFIGFILE"))
 
-	configstore, err := cfgstore.NewJSON(configfile, nil)
+	diskfs, err := fs.NewDiskFilesystem(fs.DiskConfig{})
+	if err != nil {
+		logger.Error().WithError(err).Log("Access disk filesystem failed")
+		os.Exit(1)
+	}
+
+	configstore, err := cfgstore.NewJSON(diskfs, configfile, nil)
 	if err != nil {
 		logger.Error().WithError(err).Log("Loading configuration failed")
 		os.Exit(1)
 	}
 
-	if err := doImport(logger, configstore); err != nil {
+	if err := doImport(logger, diskfs, configstore); err != nil {
 		os.Exit(1)
 	}
 }
 
-func doImport(logger log.Logger, configstore cfgstore.Store) error {
+func doImport(logger log.Logger, fs fs.Filesystem, configstore cfgstore.Store) error {
 	if logger == nil {
 		logger = log.New("")
 	}
@@ -67,23 +74,27 @@ func doImport(logger log.Logger, configstore cfgstore.Store) error {
 
 	logger = logger.WithField("database", v1filename)
 
-	if _, err := os.Stat(v1filename); err != nil {
+	if _, err := fs.Stat(v1filename); err != nil {
 		if os.IsNotExist(err) {
 			logger.Info().Log("Database doesn't exist and nothing will be imported")
 			return nil
 		}
 
 		logger.Error().WithError(err).Log("Checking for v1 database")
-
 		return fmt.Errorf("checking for v1 database: %w", err)
 	}
 
 	logger.Info().Log("Found database")
 
 	// Load an existing DB
-	datastore := store.NewJSONStore(store.JSONConfig{
-		Filepath: cfg.DB.Dir + "/db.json",
+	datastore, err := store.NewJSON(store.JSONConfig{
+		Filesystem: fs,
+		Filepath:   cfg.DB.Dir + "/db.json",
 	})
+	if err != nil {
+		logger.Error().WithError(err).Log("Creating datastore for new database failed")
+		return fmt.Errorf("creating datastore for new database failed: %w", err)
+	}
 
 	data, err := datastore.Load()
 	if err != nil {
@@ -105,7 +116,7 @@ func doImport(logger log.Logger, configstore cfgstore.Store) error {
 	importConfig.binary = cfg.FFmpeg.Binary
 
 	// Rewrite the old database to the new database
-	r, err := importV1(v1filename, importConfig)
+	r, err := importV1(fs, v1filename, importConfig)
 	if err != nil {
 		logger.Error().WithError(err).Log("Importing database failed")
 		return fmt.Errorf("importing database failed: %w", err)

app/import/main_test.go

@@ -1,20 +1,29 @@
 package main
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/datarhei/core/v16/config/store"
+	"github.com/datarhei/core/v16/io/fs"
+
 	"github.com/stretchr/testify/require"
 )
 
 func TestImport(t *testing.T) {
-	configstore := store.NewDummy()
+	memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
+	require.NoError(t, err)
+
+	memfs.WriteFileReader("/mime.types", strings.NewReader("foobar"))
+
+	configstore, err := store.NewJSON(memfs, "/config.json", nil)
+	require.NoError(t, err)
 
 	cfg := configstore.Get()
 
-	err := configstore.Set(cfg)
+	err = configstore.Set(cfg)
 	require.NoError(t, err)
 
-	err = doImport(nil, configstore)
+	err = doImport(nil, memfs, configstore)
 	require.NoError(t, err)
 }

config/config.go

@@ -9,6 +9,7 @@ import (
 	"github.com/datarhei/core/v16/config/copy"
 	"github.com/datarhei/core/v16/config/value"
 	"github.com/datarhei/core/v16/config/vars"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/math/rand"
 
 	haikunator "github.com/atrox/haikunatorgo/v2"
@@ -46,14 +47,21 @@ const version int64 = 3
 
 // Config is a wrapper for Data
 type Config struct {
+	fs   fs.Filesystem
 	vars vars.Variables
 
 	Data
 }
 
 // New returns a Config which is initialized with its default values
-func New() *Config {
-	config := &Config{}
+func New(f fs.Filesystem) *Config {
+	config := &Config{
+		fs: f,
+	}
+
+	if config.fs == nil {
+		config.fs, _ = fs.NewMemFilesystem(fs.MemConfig{})
+	}
 
 	config.init()
 
@@ -70,7 +78,7 @@ func (d *Config) Set(name, val string) error {
 
 // NewConfigFrom returns a clone of a Config
 func (d *Config) Clone() *Config {
-	data := New()
+	data := New(d.fs)
 
 	data.CreatedAt = d.CreatedAt
 	data.LoadedAt = d.LoadedAt
@@ -145,7 +153,7 @@ func (d *Config) init() {
 	d.vars.Register(value.NewInt(&d.Log.MaxLines, 1000), "log.max_lines", "CORE_LOG_MAXLINES", nil, "Number of latest log lines to keep in memory", false, false)
 
 	// DB
-	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config"), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
+	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config", d.fs), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
 
 	// Host
 	d.vars.Register(value.NewStringList(&d.Host.Name, []string{}, ","), "host.name", "CORE_HOST_NAME", nil, "Comma separated list of public host/domain names or IPs", false, false)
@@ -174,14 +182,14 @@ func (d *Config) init() {
 	d.vars.Register(value.NewBool(&d.TLS.Enable, false), "tls.enable", "CORE_TLS_ENABLE", nil, "Enable HTTPS", false, false)
 	d.vars.Register(value.NewBool(&d.TLS.Auto, false), "tls.auto", "CORE_TLS_AUTO", nil, "Enable Let's Encrypt certificate", false, false)
 	d.vars.Register(value.NewEmail(&d.TLS.Email, "cert@datarhei.com"), "tls.email", "CORE_TLS_EMAIL", nil, "Email for Let's Encrypt registration", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.CertFile, ""), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.KeyFile, ""), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.CertFile, "", d.fs), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.KeyFile, "", d.fs), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
 
 	// Storage
-	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types"), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
+	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types", d.fs), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
 
 	// Storage (Disk)
-	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data"), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
+	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data", d.fs), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
 	d.vars.Register(value.NewInt64(&d.Storage.Disk.Size, 0), "storage.disk.max_size_mbytes", "CORE_STORAGE_DISK_MAXSIZEMBYTES", nil, "Max. allowed megabytes for storage.disk.dir, 0 for unlimited", false, false)
 	d.vars.Register(value.NewBool(&d.Storage.Disk.Cache.Enable, true), "storage.disk.cache.enable", "CORE_STORAGE_DISK_CACHE_ENABLE", nil, "Enable cache for /", false, false)
 	d.vars.Register(value.NewUint64(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false)
@@ -262,7 +270,7 @@ func (d *Config) init() {
 	// Router
 	d.vars.Register(value.NewStringList(&d.Router.BlockedPrefixes, []string{"/api"}, ","), "router.blocked_prefixes", "CORE_ROUTER_BLOCKED_PREFIXES", nil, "List of path prefixes that can't be routed", false, false)
 	d.vars.Register(value.NewStringMapString(&d.Router.Routes, nil), "router.routes", "CORE_ROUTER_ROUTES", nil, "List of route mappings", false, false)
-	d.vars.Register(value.NewDir(&d.Router.UIPath, ""), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
+	d.vars.Register(value.NewDir(&d.Router.UIPath, "", d.fs), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
 }
 
 // Validate validates the current state of the Config for completeness and sanity. Errors are

config/config_test.go

@@ -1,13 +1,18 @@
 package config
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/datarhei/core/v16/config/vars"
+	"github.com/datarhei/core/v16/io/fs"
+
 	"github.com/stretchr/testify/require"
 )
 
 func TestConfigCopy(t *testing.T) {
-	config1 := New()
+	fs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+	config1 := New(fs)
 
 	config1.Version = 42
 	config1.DB.Dir = "foo"
@@ -50,3 +55,25 @@ func TestConfigCopy(t *testing.T) {
 	require.Equal(t, []string{"bar.com"}, config1.Host.Name)
 	require.Equal(t, []string{"foo.com"}, config2.Host.Name)
 }
+
+func TestValidateDefault(t *testing.T) {
+	fs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+	size, fresh, err := fs.WriteFileReader("./mime.types", strings.NewReader("xxxxx"))
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, fresh)
+	require.NoError(t, err)
+
+	cfg := New(fs)
+
+	cfg.Validate(true)
+
+	errors := []string{}
+	cfg.Messages(func(level string, v vars.Variable, message string) {
+		if level == "error" {
+			errors = append(errors, message)
+		}
+	})
+
+	require.Equal(t, 0, len(cfg.Overrides()))
+	require.Equal(t, false, cfg.HasErrors(), errors)
+}

config/data.go

@@ -6,6 +6,7 @@ import (
 	"github.com/datarhei/core/v16/config/copy"
 	v2 "github.com/datarhei/core/v16/config/v2"
 	"github.com/datarhei/core/v16/config/value"
+	"github.com/datarhei/core/v16/io/fs"
 )
 
 // Data is the actual configuration data for the app
@@ -167,8 +168,8 @@ type Data struct {
 	} `json:"router"`
 }
 
-func UpgradeV2ToV3(d *v2.Data) (*Data, error) {
-	cfg := New()
+func UpgradeV2ToV3(d *v2.Data, fs fs.Filesystem) (*Data, error) {
+	cfg := New(fs)
 
 	return MergeV2toV3(&cfg.Data, d)
 }

config/data_test.go

@@ -0,0 +1,36 @@
+package config
+
+import (
+	"testing"
+
+	v2 "github.com/datarhei/core/v16/config/v2"
+	"github.com/datarhei/core/v16/io/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpgrade(t *testing.T) {
+	fs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+
+	v2cfg := v2.New(fs)
+	v2cfg.Storage.Disk.Cache.Types = []string{".foo", ".bar"}
+
+	v3cfg, err := UpgradeV2ToV3(&v2cfg.Data, fs)
+
+	require.NoError(t, err)
+	require.Equal(t, int64(3), v3cfg.Version)
+	require.ElementsMatch(t, []string{".foo", ".bar"}, v3cfg.Storage.Disk.Cache.Types.Allow)
+	require.ElementsMatch(t, []string{".m3u8", ".mpd"}, v3cfg.Storage.Disk.Cache.Types.Block)
+}
+
+func TestDowngrade(t *testing.T) {
+	fs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+
+	v3cfg := New(fs)
+	v3cfg.Storage.Disk.Cache.Types.Allow = []string{".foo", ".bar"}
+
+	v2cfg, err := DowngradeV3toV2(&v3cfg.Data)
+
+	require.NoError(t, err)
+	require.Equal(t, int64(2), v2cfg.Version)
+	require.ElementsMatch(t, []string{".foo", ".bar"}, v2cfg.Storage.Disk.Cache.Types)
+}

config/store/dummy.go

@@ -1,73 +0,0 @@
-package store
-
-import (
-	"fmt"
-
-	"github.com/datarhei/core/v16/config"
-)
-
-type dummyStore struct {
-	current *config.Config
-	active  *config.Config
-}
-
-// NewDummyStore returns a store that returns the default config
-func NewDummy() Store {
-	s := &dummyStore{}
-
-	cfg := config.New()
-
-	cfg.DB.Dir = "."
-	cfg.FFmpeg.Binary = "true"
-	cfg.Storage.Disk.Dir = "."
-	cfg.Storage.MimeTypes = ""
-
-	s.current = cfg
-
-	cfg = config.New()
-
-	cfg.DB.Dir = "."
-	cfg.FFmpeg.Binary = "true"
-	cfg.Storage.Disk.Dir = "."
-	cfg.Storage.MimeTypes = ""
-
-	s.active = cfg
-
-	return s
-}
-
-func (c *dummyStore) Get() *config.Config {
-	return c.current.Clone()
-}
-
-func (c *dummyStore) Set(d *config.Config) error {
-	d.Validate(true)
-
-	if d.HasErrors() {
-		return fmt.Errorf("configuration data has errors after validation")
-	}
-
-	c.current = d.Clone()
-
-	return nil
-}
-
-func (c *dummyStore) GetActive() *config.Config {
-	return c.active.Clone()
-}
-
-func (c *dummyStore) SetActive(d *config.Config) error {
-	d.Validate(true)
-
-	if d.HasErrors() {
-		return fmt.Errorf("configuration data has errors after validation")
-	}
-
-	c.active = d.Clone()
-
-	return nil
-}
-
-func (c *dummyStore) Reload() error {
-	return nil
-}

config/store/json.go

@@ -10,10 +10,11 @@ import (
 	v1 "github.com/datarhei/core/v16/config/v1"
 	v2 "github.com/datarhei/core/v16/config/v2"
 	"github.com/datarhei/core/v16/encoding/json"
-	"github.com/datarhei/core/v16/io/file"
+	"github.com/datarhei/core/v16/io/fs"
 )
 
 type jsonStore struct {
+	fs   fs.Filesystem
 	path string
 
 	data map[string]*config.Config
@@ -21,18 +22,32 @@ type jsonStore struct {
 	reloadFn func()
 }
 
-// NewJSONStore will read a JSON config file from the given path. After successfully reading it in, it will be written
-// back to the path. The returned error will be nil if everything went fine.
-// If the path doesn't exist, a default JSON config file will be written to that path.
-// The returned ConfigStore can be used to retrieve or write the config.
-func NewJSON(path string, reloadFn func()) (Store, error) {
+// NewJSONStore will read the JSON config file from the given path. After successfully reading it in, it will be written
+// back to the path. The returned error will be nil if everything went fine. If the path doesn't exist, a default JSON
+// config file will be written to that path. The returned ConfigStore can be used to retrieve or write the config.
+func NewJSON(f fs.Filesystem, path string, reloadFn func()) (Store, error) {
 	c := &jsonStore{
-		path:     path,
+		fs:       f,
 		data:     make(map[string]*config.Config),
 		reloadFn: reloadFn,
 	}
 
-	c.data["base"] = config.New()
+	path, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine absolute path of '%s': %w", path, err)
+	}
+
+	c.path = path
+
+	if len(c.path) == 0 {
+		c.path = "/config.json"
+	}
+
+	if c.fs == nil {
+		return nil, fmt.Errorf("no valid filesystem provided")
+	}
+
+	c.data["base"] = config.New(f)
 
 	if err := c.load(c.data["base"]); err != nil {
 		return nil, fmt.Errorf("failed to read JSON from '%s': %w", path, err)
@@ -106,11 +121,11 @@ func (c *jsonStore) load(cfg *config.Config) error {
 		return nil
 	}
 
-	if _, err := os.Stat(c.path); os.IsNotExist(err) {
+	if _, err := c.fs.Stat(c.path); os.IsNotExist(err) {
 		return nil
 	}
 
-	jsondata, err := os.ReadFile(c.path)
+	jsondata, err := c.fs.ReadFile(c.path)
 	if err != nil {
 		return err
 	}
@@ -141,28 +156,9 @@ func (c *jsonStore) store(data *config.Config) error {
 		return err
 	}
 
-	dir, filename := filepath.Split(c.path)
+	_, _, err = c.fs.WriteFileSafe(c.path, jsondata)
 
-	tmpfile, err := os.CreateTemp(dir, filename)
-	if err != nil {
-		return err
-	}
-
-	defer os.Remove(tmpfile.Name())
-
-	if _, err := tmpfile.Write(jsondata); err != nil {
-		return err
-	}
-
-	if err := tmpfile.Close(); err != nil {
-		return err
-	}
-
-	if err := file.Rename(tmpfile.Name(), c.path); err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func migrate(jsondata []byte) (*config.Data, error) {
@@ -174,38 +170,38 @@ func migrate(jsondata []byte) (*config.Data, error) {
 	}
 
 	if version.Version == 1 {
-		dataV1 := &v1.New().Data
+		dataV1 := &v1.New(nil).Data
 
 		if err := gojson.Unmarshal(jsondata, dataV1); err != nil {
 			return nil, json.FormatError(jsondata, err)
 		}
 
-		dataV2, err := v2.UpgradeV1ToV2(dataV1)
+		dataV2, err := v2.UpgradeV1ToV2(dataV1, nil)
 		if err != nil {
 			return nil, err
 		}
 
-		dataV3, err := config.UpgradeV2ToV3(dataV2)
+		dataV3, err := config.UpgradeV2ToV3(dataV2, nil)
 		if err != nil {
 			return nil, err
 		}
 
 		data = dataV3
 	} else if version.Version == 2 {
-		dataV2 := &v2.New().Data
+		dataV2 := &v2.New(nil).Data
 
 		if err := gojson.Unmarshal(jsondata, dataV2); err != nil {
 			return nil, json.FormatError(jsondata, err)
 		}
 
-		dataV3, err := config.UpgradeV2ToV3(dataV2)
+		dataV3, err := config.UpgradeV2ToV3(dataV2, nil)
 		if err != nil {
 			return nil, err
 		}
 
 		data = dataV3
 	} else if version.Version == 3 {
-		dataV3 := &config.New().Data
+		dataV3 := &config.New(nil).Data
 
 		if err := gojson.Unmarshal(jsondata, dataV3); err != nil {
 			return nil, json.FormatError(jsondata, err)

config/store/json_test.go

@@ -18,7 +18,7 @@ func TestMigrationV1ToV3(t *testing.T) {
 	jsondatav3, err := os.ReadFile("./fixtures/config_v1_v3.json")
 	require.NoError(t, err)
 
-	datav3 := config.New()
+	datav3 := config.New(nil)
 	json.Unmarshal(jsondatav3, datav3)
 
 	data, err := migrate(jsondatav1)
@@ -37,7 +37,7 @@ func TestMigrationV2ToV3(t *testing.T) {
 	jsondatav3, err := os.ReadFile("./fixtures/config_v2_v3.json")
 	require.NoError(t, err)
 
-	datav3 := config.New()
+	datav3 := config.New(nil)
 	json.Unmarshal(jsondatav3, datav3)
 
 	data, err := migrate(jsondatav2)

config/v1/config.go

@@ -8,6 +8,7 @@ import (
 	"github.com/datarhei/core/v16/config/copy"
 	"github.com/datarhei/core/v16/config/value"
 	"github.com/datarhei/core/v16/config/vars"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/math/rand"
 
 	haikunator "github.com/atrox/haikunatorgo/v2"
@@ -21,14 +22,21 @@ const version int64 = 1
 
 // Config is a wrapper for Data
 type Config struct {
+	fs   fs.Filesystem
 	vars vars.Variables
 
 	Data
 }
 
 // New returns a Config which is initialized with its default values
-func New() *Config {
-	cfg := &Config{}
+func New(f fs.Filesystem) *Config {
+	cfg := &Config{
+		fs: f,
+	}
+
+	if cfg.fs == nil {
+		cfg.fs, _ = fs.NewMemFilesystem(fs.MemConfig{})
+	}
 
 	cfg.init()
 
@@ -45,7 +53,7 @@ func (d *Config) Set(name, val string) error {
 
 // NewConfigFrom returns a clone of a Config
 func (d *Config) Clone() *Config {
-	data := New()
+	data := New(d.fs)
 
 	data.CreatedAt = d.CreatedAt
 	data.LoadedAt = d.LoadedAt
@@ -118,7 +126,7 @@ func (d *Config) init() {
 	d.vars.Register(value.NewInt(&d.Log.MaxLines, 1000), "log.max_lines", "CORE_LOG_MAXLINES", nil, "Number of latest log lines to keep in memory", false, false)
 
 	// DB
-	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config"), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
+	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config", d.fs), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
 
 	// Host
 	d.vars.Register(value.NewStringList(&d.Host.Name, []string{}, ","), "host.name", "CORE_HOST_NAME", nil, "Comma separated list of public host/domain names or IPs", false, false)
@@ -146,14 +154,14 @@ func (d *Config) init() {
 	d.vars.Register(value.NewAddress(&d.TLS.Address, ":8181"), "tls.address", "CORE_TLS_ADDRESS", nil, "HTTPS listening address", false, false)
 	d.vars.Register(value.NewBool(&d.TLS.Enable, false), "tls.enable", "CORE_TLS_ENABLE", nil, "Enable HTTPS", false, false)
 	d.vars.Register(value.NewBool(&d.TLS.Auto, false), "tls.auto", "CORE_TLS_AUTO", nil, "Enable Let's Encrypt certificate", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.CertFile, ""), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.KeyFile, ""), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.CertFile, "", d.fs), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.KeyFile, "", d.fs), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
 
 	// Storage
-	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types"), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
+	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types", d.fs), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
 
 	// Storage (Disk)
-	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data"), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
+	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data", d.fs), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
 	d.vars.Register(value.NewInt64(&d.Storage.Disk.Size, 0), "storage.disk.max_size_mbytes", "CORE_STORAGE_DISK_MAXSIZEMBYTES", nil, "Max. allowed megabytes for storage.disk.dir, 0 for unlimited", false, false)
 	d.vars.Register(value.NewBool(&d.Storage.Disk.Cache.Enable, true), "storage.disk.cache.enable", "CORE_STORAGE_DISK_CACHE_ENABLE", nil, "Enable cache for /", false, false)
 	d.vars.Register(value.NewUint64(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false)
@@ -228,7 +236,7 @@ func (d *Config) init() {
 	// Router
 	d.vars.Register(value.NewStringList(&d.Router.BlockedPrefixes, []string{"/api"}, ","), "router.blocked_prefixes", "CORE_ROUTER_BLOCKED_PREFIXES", nil, "List of path prefixes that can't be routed", false, false)
 	d.vars.Register(value.NewStringMapString(&d.Router.Routes, nil), "router.routes", "CORE_ROUTER_ROUTES", nil, "List of route mappings", false, false)
-	d.vars.Register(value.NewDir(&d.Router.UIPath, ""), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
+	d.vars.Register(value.NewDir(&d.Router.UIPath, "", d.fs), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
 }
 
 // Validate validates the current state of the Config for completeness and sanity. Errors are

config/v2/config.go

@@ -8,6 +8,7 @@ import (
 	"github.com/datarhei/core/v16/config/copy"
 	"github.com/datarhei/core/v16/config/value"
 	"github.com/datarhei/core/v16/config/vars"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/math/rand"
 
 	haikunator "github.com/atrox/haikunatorgo/v2"
@@ -21,14 +22,21 @@ const version int64 = 2
 
 // Config is a wrapper for Data
 type Config struct {
+	fs   fs.Filesystem
 	vars vars.Variables
 
 	Data
 }
 
 // New returns a Config which is initialized with its default values
-func New() *Config {
-	cfg := &Config{}
+func New(f fs.Filesystem) *Config {
+	cfg := &Config{
+		fs: f,
+	}
+
+	if cfg.fs == nil {
+		cfg.fs, _ = fs.NewMemFilesystem(fs.MemConfig{})
+	}
 
 	cfg.init()
 
@@ -45,7 +53,7 @@ func (d *Config) Set(name, val string) error {
 
 // NewConfigFrom returns a clone of a Config
 func (d *Config) Clone() *Config {
-	data := New()
+	data := New(d.fs)
 
 	data.CreatedAt = d.CreatedAt
 	data.LoadedAt = d.LoadedAt
@@ -118,7 +126,7 @@ func (d *Config) init() {
 	d.vars.Register(value.NewInt(&d.Log.MaxLines, 1000), "log.max_lines", "CORE_LOG_MAXLINES", nil, "Number of latest log lines to keep in memory", false, false)
 
 	// DB
-	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config"), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
+	d.vars.Register(value.NewMustDir(&d.DB.Dir, "./config", d.fs), "db.dir", "CORE_DB_DIR", nil, "Directory for holding the operational data", false, false)
 
 	// Host
 	d.vars.Register(value.NewStringList(&d.Host.Name, []string{}, ","), "host.name", "CORE_HOST_NAME", nil, "Comma separated list of public host/domain names or IPs", false, false)
@@ -146,14 +154,14 @@ func (d *Config) init() {
 	d.vars.Register(value.NewAddress(&d.TLS.Address, ":8181"), "tls.address", "CORE_TLS_ADDRESS", nil, "HTTPS listening address", false, false)
 	d.vars.Register(value.NewBool(&d.TLS.Enable, false), "tls.enable", "CORE_TLS_ENABLE", nil, "Enable HTTPS", false, false)
 	d.vars.Register(value.NewBool(&d.TLS.Auto, false), "tls.auto", "CORE_TLS_AUTO", nil, "Enable Let's Encrypt certificate", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.CertFile, ""), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
-	d.vars.Register(value.NewFile(&d.TLS.KeyFile, ""), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.CertFile, "", d.fs), "tls.cert_file", "CORE_TLS_CERTFILE", nil, "Path to certificate file in PEM format", false, false)
+	d.vars.Register(value.NewFile(&d.TLS.KeyFile, "", d.fs), "tls.key_file", "CORE_TLS_KEYFILE", nil, "Path to key file in PEM format", false, false)
 
 	// Storage
-	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types"), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
+	d.vars.Register(value.NewFile(&d.Storage.MimeTypes, "./mime.types", d.fs), "storage.mimetypes_file", "CORE_STORAGE_MIMETYPES_FILE", []string{"CORE_MIMETYPES_FILE"}, "Path to file with mime-types", false, false)
 
 	// Storage (Disk)
-	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data"), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
+	d.vars.Register(value.NewMustDir(&d.Storage.Disk.Dir, "./data", d.fs), "storage.disk.dir", "CORE_STORAGE_DISK_DIR", nil, "Directory on disk, exposed on /", false, false)
 	d.vars.Register(value.NewInt64(&d.Storage.Disk.Size, 0), "storage.disk.max_size_mbytes", "CORE_STORAGE_DISK_MAXSIZEMBYTES", nil, "Max. allowed megabytes for storage.disk.dir, 0 for unlimited", false, false)
 	d.vars.Register(value.NewBool(&d.Storage.Disk.Cache.Enable, true), "storage.disk.cache.enable", "CORE_STORAGE_DISK_CACHE_ENABLE", nil, "Enable cache for /", false, false)
 	d.vars.Register(value.NewUint64(&d.Storage.Disk.Cache.Size, 0), "storage.disk.cache.max_size_mbytes", "CORE_STORAGE_DISK_CACHE_MAXSIZEMBYTES", nil, "Max. allowed cache size, 0 for unlimited", false, false)
@@ -229,7 +237,7 @@ func (d *Config) init() {
 	// Router
 	d.vars.Register(value.NewStringList(&d.Router.BlockedPrefixes, []string{"/api"}, ","), "router.blocked_prefixes", "CORE_ROUTER_BLOCKED_PREFIXES", nil, "List of path prefixes that can't be routed", false, false)
 	d.vars.Register(value.NewStringMapString(&d.Router.Routes, nil), "router.routes", "CORE_ROUTER_ROUTES", nil, "List of route mappings", false, false)
-	d.vars.Register(value.NewDir(&d.Router.UIPath, ""), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
+	d.vars.Register(value.NewDir(&d.Router.UIPath, "", d.fs), "router.ui_path", "CORE_ROUTER_UI_PATH", nil, "Path to a directory holding UI files mounted as /ui", false, false)
 }
 
 // Validate validates the current state of the Config for completeness and sanity. Errors are

config/v2/data.go

@@ -10,6 +10,7 @@ import (
 	"github.com/datarhei/core/v16/config/copy"
 	v1 "github.com/datarhei/core/v16/config/v1"
 	"github.com/datarhei/core/v16/config/value"
+	"github.com/datarhei/core/v16/io/fs"
 )
 
 type Data struct {
@@ -164,8 +165,8 @@ type Data struct {
 	} `json:"router"`
 }
 
-func UpgradeV1ToV2(d *v1.Data) (*Data, error) {
-	cfg := New()
+func UpgradeV1ToV2(d *v1.Data, fs fs.Filesystem) (*Data, error) {
+	cfg := New(fs)
 
 	return MergeV1ToV2(&cfg.Data, d)
 }

config/value/os.go

@@ -2,43 +2,52 @@ package value
 
 import (
 	"fmt"
-	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+
+	"github.com/datarhei/core/v16/io/fs"
 )
 
 // must directory
 
-type MustDir string
+type MustDir struct {
+	p  *string
+	fs fs.Filesystem
+}
+
+func NewMustDir(p *string, val string, fs fs.Filesystem) *MustDir {
+	v := &MustDir{
+		p:  p,
+		fs: fs,
+	}
 
-func NewMustDir(p *string, val string) *MustDir {
 	*p = val
 
-	return (*MustDir)(p)
+	return v
 }
 
 func (u *MustDir) Set(val string) error {
-	*u = MustDir(val)
+	*u.p = val
 	return nil
 }
 
 func (u *MustDir) String() string {
-	return string(*u)
+	return *u.p
 }
 
 func (u *MustDir) Validate() error {
-	val := string(*u)
+	val := *u.p
 
 	if len(strings.TrimSpace(val)) == 0 {
 		return fmt.Errorf("path name must not be empty")
 	}
 
-	if err := os.MkdirAll(val, 0750); err != nil {
+	if err := u.fs.MkdirAll(val, 0750); err != nil {
 		return fmt.Errorf("%s can't be created (%w)", val, err)
 	}
 
-	finfo, err := os.Stat(val)
+	finfo, err := u.fs.Stat(val)
 	if err != nil {
 		return fmt.Errorf("%s does not exist", val)
 	}
@@ -51,36 +60,44 @@ func (u *MustDir) Validate() error {
 }
 
 func (u *MustDir) IsEmpty() bool {
-	return len(string(*u)) == 0
+	return len(*u.p) == 0
 }
 
 // directory
 
-type Dir string
+type Dir struct {
+	p  *string
+	fs fs.Filesystem
+}
+
+func NewDir(p *string, val string, fs fs.Filesystem) *Dir {
+	v := &Dir{
+		p:  p,
+		fs: fs,
+	}
 
-func NewDir(p *string, val string) *Dir {
 	*p = val
 
-	return (*Dir)(p)
+	return v
 }
 
 func (u *Dir) Set(val string) error {
-	*u = Dir(val)
+	*u.p = val
 	return nil
 }
 
 func (u *Dir) String() string {
-	return string(*u)
+	return *u.p
 }
 
 func (u *Dir) Validate() error {
-	val := string(*u)
+	val := *u.p
 
 	if len(strings.TrimSpace(val)) == 0 {
 		return nil
 	}
 
-	finfo, err := os.Stat(val)
+	finfo, err := u.fs.Stat(val)
 	if err != nil {
 		return fmt.Errorf("%s does not exist", val)
 	}
@@ -93,7 +110,7 @@ func (u *Dir) Validate() error {
 }
 
 func (u *Dir) IsEmpty() bool {
-	return len(string(*u)) == 0
+	return len(*u.p) == 0
 }
 
 // executable
@@ -132,31 +149,39 @@ func (u *Exec) IsEmpty() bool {
 
 // regular file
 
-type File string
+type File struct {
+	p  *string
+	fs fs.Filesystem
+}
+
+func NewFile(p *string, val string, fs fs.Filesystem) *File {
+	v := &File{
+		p:  p,
+		fs: fs,
+	}
 
-func NewFile(p *string, val string) *File {
 	*p = val
 
-	return (*File)(p)
+	return v
 }
 
 func (u *File) Set(val string) error {
-	*u = File(val)
+	*u.p = val
 	return nil
 }
 
 func (u *File) String() string {
-	return string(*u)
+	return *u.p
 }
 
 func (u *File) Validate() error {
-	val := string(*u)
+	val := *u.p
 
 	if len(val) == 0 {
 		return nil
 	}
 
-	finfo, err := os.Stat(val)
+	finfo, err := u.fs.Stat(val)
 	if err != nil {
 		return fmt.Errorf("%s does not exist", val)
 	}
@@ -169,7 +194,7 @@ func (u *File) Validate() error {
 }
 
 func (u *File) IsEmpty() bool {
-	return len(string(*u)) == 0
+	return len(*u.p) == 0
 }
 
 // absolute path

config/value/primitives.go

@@ -1,6 +1,7 @@
 package value
 
 import (
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -127,11 +128,20 @@ func (s *StringMapString) String() string {
 		return "(empty)"
 	}
 
+	sms := *s.p
+
+	keys := []string{}
+	for k := range sms {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
 	mappings := make([]string, len(*s.p))
 
 	i := 0
-	for k, v := range *s.p {
-		mappings[i] = k + ":" + v
+	for _, k := range keys {
+		mappings[i] = k + ":" + sms[k]
 		i++
 	}
 

docs/docs.go

@@ -337,452 +337,6 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/v3/fs/disk": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "List all files on the filesystem",
-                "operationId": "diskfs-3-list-files",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "glob pattern for file names",
-                        "name": "glob",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "none, name, size, lastmod",
-                        "name": "sort",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "asc, desc",
-                        "name": "order",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.FileInfo"
-                            }
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/disk/{path}": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Fetch a file from the filesystem. The contents of that file are returned.",
-                "produces": [
-                    "application/data",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Fetch a file from the filesystem",
-                "operationId": "diskfs-3-get-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "file"
-                        }
-                    },
-                    "301": {
-                        "description": "Moved Permanently",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "put": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Writes or overwrites a file on the filesystem",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Add a file to the filesystem",
-                "operationId": "diskfs-3-put-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "File data",
-                        "name": "data",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "type": "integer"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "204": {
-                        "description": "No Content",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "507": {
-                        "description": "Insufficient Storage",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Remove a file from the filesystem",
-                "produces": [
-                    "text/plain"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Remove a file from the filesystem",
-                "operationId": "diskfs-3-delete-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/mem": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "List all files on the memory filesystem",
-                "operationId": "memfs-3-list-files",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "glob pattern for file names",
-                        "name": "glob",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "none, name, size, lastmod",
-                        "name": "sort",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "asc, desc",
-                        "name": "order",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.FileInfo"
-                            }
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/mem/{path}": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Fetch a file from the memory filesystem",
-                "produces": [
-                    "application/data",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Fetch a file from the memory filesystem",
-                "operationId": "memfs-3-get-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "file"
-                        }
-                    },
-                    "301": {
-                        "description": "Moved Permanently",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "put": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Writes or overwrites a file on the memory filesystem",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Add a file to the memory filesystem",
-                "operationId": "memfs-3-put-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "File data",
-                        "name": "data",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "type": "integer"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "204": {
-                        "description": "No Content",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "507": {
-                        "description": "Insufficient Storage",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Remove a file from the memory filesystem",
-                "produces": [
-                    "text/plain"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Remove a file from the memory filesystem",
-                "operationId": "memfs-3-delete-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "patch": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Create a link to a file in the memory filesystem",
-                "operationId": "memfs-3-patch",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "Path to the file to link to",
-                        "name": "url",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "string"
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            }
-        },
         "/api/v3/fs/{name}": {
             "get": {
                 "security": [

docs/swagger.json

@@ -329,452 +329,6 @@
                 }
             }
         },
-        "/api/v3/fs/disk": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "List all files on the filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "List all files on the filesystem",
-                "operationId": "diskfs-3-list-files",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "glob pattern for file names",
-                        "name": "glob",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "none, name, size, lastmod",
-                        "name": "sort",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "asc, desc",
-                        "name": "order",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.FileInfo"
-                            }
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/disk/{path}": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Fetch a file from the filesystem. The contents of that file are returned.",
-                "produces": [
-                    "application/data",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Fetch a file from the filesystem",
-                "operationId": "diskfs-3-get-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "file"
-                        }
-                    },
-                    "301": {
-                        "description": "Moved Permanently",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "put": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Writes or overwrites a file on the filesystem",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Add a file to the filesystem",
-                "operationId": "diskfs-3-put-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "File data",
-                        "name": "data",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "type": "integer"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "204": {
-                        "description": "No Content",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "507": {
-                        "description": "Insufficient Storage",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Remove a file from the filesystem",
-                "produces": [
-                    "text/plain"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Remove a file from the filesystem",
-                "operationId": "diskfs-3-delete-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/mem": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "List all files on the memory filesystem. The listing can be ordered by name, size, or date of last modification in ascending or descending order.",
-                "produces": [
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "List all files on the memory filesystem",
-                "operationId": "memfs-3-list-files",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "glob pattern for file names",
-                        "name": "glob",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "none, name, size, lastmod",
-                        "name": "sort",
-                        "in": "query"
-                    },
-                    {
-                        "type": "string",
-                        "description": "asc, desc",
-                        "name": "order",
-                        "in": "query"
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "$ref": "#/definitions/api.FileInfo"
-                            }
-                        }
-                    }
-                }
-            }
-        },
-        "/api/v3/fs/mem/{path}": {
-            "get": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Fetch a file from the memory filesystem",
-                "produces": [
-                    "application/data",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Fetch a file from the memory filesystem",
-                "operationId": "memfs-3-get-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "file"
-                        }
-                    },
-                    "301": {
-                        "description": "Moved Permanently",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "put": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Writes or overwrites a file on the memory filesystem",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Add a file to the memory filesystem",
-                "operationId": "memfs-3-put-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "File data",
-                        "name": "data",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "array",
-                            "items": {
-                                "type": "integer"
-                            }
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "204": {
-                        "description": "No Content",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "507": {
-                        "description": "Insufficient Storage",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "delete": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Remove a file from the memory filesystem",
-                "produces": [
-                    "text/plain"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Remove a file from the memory filesystem",
-                "operationId": "memfs-3-delete-file",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            },
-            "patch": {
-                "security": [
-                    {
-                        "ApiKeyAuth": []
-                    }
-                ],
-                "description": "Create a link to a file in the memory filesystem. The file linked to has to exist.",
-                "consumes": [
-                    "application/data"
-                ],
-                "produces": [
-                    "text/plain",
-                    "application/json"
-                ],
-                "tags": [
-                    "v16.7.2"
-                ],
-                "summary": "Create a link to a file in the memory filesystem",
-                "operationId": "memfs-3-patch",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Path to file",
-                        "name": "path",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "description": "Path to the file to link to",
-                        "name": "url",
-                        "in": "body",
-                        "required": true,
-                        "schema": {
-                            "type": "string"
-                        }
-                    }
-                ],
-                "responses": {
-                    "201": {
-                        "description": "Created",
-                        "schema": {
-                            "type": "string"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/api.Error"
-                        }
-                    }
-                }
-            }
-        },
         "/api/v3/fs/{name}": {
             "get": {
                 "security": [

docs/swagger.yaml

@@ -2266,298 +2266,6 @@ paths:
       security:
       - ApiKeyAuth: []
       summary: Add a file to a filesystem
-  /api/v3/fs/disk:
-    get:
-      description: List all files on the filesystem. The listing can be ordered by
-        name, size, or date of last modification in ascending or descending order.
-      operationId: diskfs-3-list-files
-      parameters:
-      - description: glob pattern for file names
-        in: query
-        name: glob
-        type: string
-      - description: none, name, size, lastmod
-        in: query
-        name: sort
-        type: string
-      - description: asc, desc
-        in: query
-        name: order
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            items:
-              $ref: '#/definitions/api.FileInfo'
-            type: array
-      security:
-      - ApiKeyAuth: []
-      summary: List all files on the filesystem
-      tags:
-      - v16.7.2
-  /api/v3/fs/disk/{path}:
-    delete:
-      description: Remove a file from the filesystem
-      operationId: diskfs-3-delete-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      produces:
-      - text/plain
-      responses:
-        "200":
-          description: OK
-          schema:
-            type: string
-        "404":
-          description: Not Found
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Remove a file from the filesystem
-      tags:
-      - v16.7.2
-    get:
-      description: Fetch a file from the filesystem. The contents of that file are
-        returned.
-      operationId: diskfs-3-get-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      produces:
-      - application/data
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            type: file
-        "301":
-          description: Moved Permanently
-          schema:
-            type: string
-        "404":
-          description: Not Found
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Fetch a file from the filesystem
-      tags:
-      - v16.7.2
-    put:
-      consumes:
-      - application/data
-      description: Writes or overwrites a file on the filesystem
-      operationId: diskfs-3-put-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      - description: File data
-        in: body
-        name: data
-        required: true
-        schema:
-          items:
-            type: integer
-          type: array
-      produces:
-      - text/plain
-      - application/json
-      responses:
-        "201":
-          description: Created
-          schema:
-            type: string
-        "204":
-          description: No Content
-          schema:
-            type: string
-        "507":
-          description: Insufficient Storage
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Add a file to the filesystem
-      tags:
-      - v16.7.2
-  /api/v3/fs/mem:
-    get:
-      description: List all files on the memory filesystem. The listing can be ordered
-        by name, size, or date of last modification in ascending or descending order.
-      operationId: memfs-3-list-files
-      parameters:
-      - description: glob pattern for file names
-        in: query
-        name: glob
-        type: string
-      - description: none, name, size, lastmod
-        in: query
-        name: sort
-        type: string
-      - description: asc, desc
-        in: query
-        name: order
-        type: string
-      produces:
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            items:
-              $ref: '#/definitions/api.FileInfo'
-            type: array
-      security:
-      - ApiKeyAuth: []
-      summary: List all files on the memory filesystem
-      tags:
-      - v16.7.2
-  /api/v3/fs/mem/{path}:
-    delete:
-      description: Remove a file from the memory filesystem
-      operationId: memfs-3-delete-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      produces:
-      - text/plain
-      responses:
-        "200":
-          description: OK
-          schema:
-            type: string
-        "404":
-          description: Not Found
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Remove a file from the memory filesystem
-      tags:
-      - v16.7.2
-    get:
-      description: Fetch a file from the memory filesystem
-      operationId: memfs-3-get-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      produces:
-      - application/data
-      - application/json
-      responses:
-        "200":
-          description: OK
-          schema:
-            type: file
-        "301":
-          description: Moved Permanently
-          schema:
-            type: string
-        "404":
-          description: Not Found
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Fetch a file from the memory filesystem
-      tags:
-      - v16.7.2
-    patch:
-      consumes:
-      - application/data
-      description: Create a link to a file in the memory filesystem. The file linked
-        to has to exist.
-      operationId: memfs-3-patch
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      - description: Path to the file to link to
-        in: body
-        name: url
-        required: true
-        schema:
-          type: string
-      produces:
-      - text/plain
-      - application/json
-      responses:
-        "201":
-          description: Created
-          schema:
-            type: string
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Create a link to a file in the memory filesystem
-      tags:
-      - v16.7.2
-    put:
-      consumes:
-      - application/data
-      description: Writes or overwrites a file on the memory filesystem
-      operationId: memfs-3-put-file
-      parameters:
-      - description: Path to file
-        in: path
-        name: path
-        required: true
-        type: string
-      - description: File data
-        in: body
-        name: data
-        required: true
-        schema:
-          items:
-            type: integer
-          type: array
-      produces:
-      - text/plain
-      - application/json
-      responses:
-        "201":
-          description: Created
-          schema:
-            type: string
-        "204":
-          description: No Content
-          schema:
-            type: string
-        "507":
-          description: Insufficient Storage
-          schema:
-            $ref: '#/definitions/api.Error'
-      security:
-      - ApiKeyAuth: []
-      summary: Add a file to the memory filesystem
-      tags:
-      - v16.7.2
   /api/v3/log:
     get:
       description: Get the last log lines of the Restreamer application

go.mod

@@ -11,25 +11,25 @@ require (
 	github.com/datarhei/joy4 v0.0.0-20220914170649-23c70d207759
 	github.com/go-playground/validator/v10 v10.11.1
 	github.com/gobwas/glob v0.2.3
-	github.com/golang-jwt/jwt/v4 v4.4.2
+	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/google/uuid v1.3.0
 	github.com/invopop/jsonschema v0.4.0
 	github.com/joho/godotenv v1.4.0
 	github.com/labstack/echo/v4 v4.9.1
 	github.com/lithammer/shortuuid/v4 v4.0.0
-	github.com/mattn/go-isatty v0.0.16
-	github.com/minio/minio-go/v7 v7.0.39
+	github.com/mattn/go-isatty v0.0.17
+	github.com/minio/minio-go/v7 v7.0.47
 	github.com/prep/average v0.0.0-20200506183628-d26c465f48c3
-	github.com/prometheus/client_golang v1.13.1
-	github.com/shirou/gopsutil/v3 v3.22.10
+	github.com/prometheus/client_golang v1.14.0
+	github.com/shirou/gopsutil/v3 v3.22.11
 	github.com/stretchr/testify v1.8.1
 	github.com/swaggo/echo-swagger v1.3.5
 	github.com/swaggo/swag v1.8.7
 	github.com/vektah/gqlparser/v2 v2.5.1
 	github.com/xeipuuv/gojsonschema v1.2.0
-	go.uber.org/zap v1.23.0
-	golang.org/x/mod v0.6.0
-	golang.org/x/net v0.1.0
+	go.uber.org/zap v1.24.0
+	golang.org/x/mod v0.7.0
+	golang.org/x/net v0.5.0
 )
 
 require (
@@ -37,14 +37,14 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/go-openapi/spec v0.20.7 // indirect
+	github.com/go-openapi/spec v0.20.8 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
@@ -55,8 +55,8 @@ require (
 	github.com/iancoleman/orderedmap v0.2.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/klauspost/cpuid/v2 v2.1.2 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/labstack/gommon v0.4.0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/libdns/libdns v0.2.1 // indirect
@@ -71,17 +71,19 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/xid v1.4.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect
-	github.com/tklauser/go-sysconf v0.3.10 // indirect
-	github.com/tklauser/numcpus v0.5.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/urfave/cli/v2 v2.8.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
@@ -90,13 +92,14 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/sys v0.1.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
-	golang.org/x/time v0.1.0 // indirect
-	golang.org/x/tools v0.2.0 // indirect
+	go.uber.org/goleak v1.1.12 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.4.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/ini.v1 v1.66.6 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,41 +1,7 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/99designs/gqlgen v0.17.20 h1:O7WzccIhKB1dm+7g6dhQcULINftfiLSBg2l/mwbpJMw=
 github.com/99designs/gqlgen v0.17.20/go.mod h1:Mja2HI23kWT1VRH09hvWshFgOzKswpO20o4ScpJIES4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -46,11 +12,6 @@ github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaW
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
@@ -61,21 +22,12 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c h1:8XZeJrs4+ZYhJeJ2aZxADI2tGADS15AzIF8MQ8XAhT4=
 github.com/benburkert/openpgp v0.0.0-20160410205803-c2471f86866c/go.mod h1:x1vxHcL/9AVzuk5HOloOEPrtJY0MaalYr78afXZ+pWI=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/caddyserver/certmagic v0.17.2 h1:o30seC1T/dBqBCNNGNHWwj2i5/I/FMjBbTAhjADP3nE=
 github.com/caddyserver/certmagic v0.17.2/go.mod h1:ouWUuC490GOLJzkyN35eXfV8bSbwMwSf4bdhkIxtdQE=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -89,24 +41,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -116,8 +53,8 @@ github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/a
 github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
 github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
 github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
-github.com/go-openapi/spec v0.20.7 h1:1Rlu/ZrOCCob0n+JKKJAWhNWMPW8bOZRg8FJaY+0SKI=
-github.com/go-openapi/spec v0.20.7/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
+github.com/go-openapi/spec v0.20.8 h1:ubHmXNY3FCIOinT8RNrrPfGc9t7I1qhPtdOGoG2AxRU=
+github.com/go-openapi/spec v0.20.8/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
@@ -130,110 +67,48 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j
 github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
 github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
-github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
-github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
+github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA=
 github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/invopop/jsonschema v0.4.0 h1:Yuy/unfgCnfV5Wl7H0HgFufp/rlurqPOOuacqyByrws=
 github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
+github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.1.2 h1:XhdX4fqAJUA0yj+kUwMavO0hHrSPAecYdYf1ZmxHvak=
-github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
+github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -268,9 +143,9 @@ github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mholt/acmez v1.0.4 h1:N3cE4Pek+dSolbsofIkAYz6H1d3pE+2G0os7QHslf80=
@@ -279,8 +154,8 @@ github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.39 h1:upnbu1jCGOqEvrGSpRauSN9ZG7RCHK7VHxXS8Vmg2zk=
-github.com/minio/minio-go/v7 v7.0.39/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs=
+github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -289,12 +164,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
@@ -302,48 +173,30 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
-github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c h1:NRoLoZvkBTKvR5gQLgA3e0hqjkY9u1wm+iOL45VN/qI=
-github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
+github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/prep/average v0.0.0-20200506183628-d26c465f48c3 h1:Y7qCvg282QmlyrVQuL2fgGwebuw7zvfnRym09r+dUGc=
 github.com/prep/average v0.0.0-20200506183628-d26c465f48c3/go.mod h1:0ZE5gcyWKS151WBDIpmLshHY0l+3edpuKnBUWVVbWKk=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c=
-github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
+github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
+github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -351,21 +204,16 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v3 v3.22.10 h1:4KMHdfBRYXGF9skjDWiL4RA2N+E8dRdodU/bOZpPoVg=
-github.com/shirou/gopsutil/v3 v3.22.10/go.mod h1:QNza6r4YQoydyCfo6rH0blGfKahgibh4dQmV5xdFkQk=
+github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3+1ndM=
+github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -382,11 +230,10 @@ github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a/go.mod h1:lKJPbtWzJ9J
 github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
 github.com/swaggo/swag v1.8.7 h1:2K9ivTD3teEO+2fXV6zrZKDqk5IuU2aJtBDo8U7omWU=
 github.com/swaggo/swag v1.8.7/go.mod h1:ezQVUUhly8dludpVk+/PuwJWvLLanB13ygV5Pr9enSk=
-github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
-github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
-github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
-github.com/tklauser/numcpus v0.5.0 h1:ooe7gN0fg6myJ0EKoTAf5hebTZrH52px3New/D9iJ+A=
-github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKoflh+RQjo=
+github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
+github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
+github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
+github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4=
 github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY=
@@ -406,180 +253,69 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
-go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
-go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
-go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
-golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -588,166 +324,49 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
-golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -755,15 +374,12 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
-gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -771,13 +387,3 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

http/handler/api/config_test.go

@@ -10,14 +10,19 @@ import (
 	"github.com/datarhei/core/v16/config/store"
 	v1 "github.com/datarhei/core/v16/config/v1"
 	"github.com/datarhei/core/v16/http/mock"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/labstack/echo/v4"
 	"github.com/stretchr/testify/require"
 )
 
-func getDummyConfigRouter() (*echo.Echo, store.Store) {
+func getDummyConfigRouter(t *testing.T) (*echo.Echo, store.Store) {
 	router := mock.DummyEcho()
 
-	config := store.NewDummy()
+	memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
+	require.NoError(t, err)
+
+	config, err := store.NewJSON(memfs, "/config.json", nil)
+	require.NoError(t, err)
 
 	handler := NewConfig(config)
 
@@ -28,7 +33,7 @@ func getDummyConfigRouter() (*echo.Echo, store.Store) {
 }
 
 func TestConfigGet(t *testing.T) {
-	router, _ := getDummyConfigRouter()
+	router, _ := getDummyConfigRouter(t)
 
 	mock.Request(t, http.StatusOK, router, "GET", "/", nil)
 
@@ -36,18 +41,18 @@ func TestConfigGet(t *testing.T) {
 }
 
 func TestConfigSetConflict(t *testing.T) {
-	router, _ := getDummyConfigRouter()
+	router, _ := getDummyConfigRouter(t)
 
 	var data bytes.Buffer
 
 	encoder := json.NewEncoder(&data)
-	encoder.Encode(config.New())
+	encoder.Encode(config.New(nil))
 
 	mock.Request(t, http.StatusConflict, router, "PUT", "/", &data)
 }
 
 func TestConfigSet(t *testing.T) {
-	router, store := getDummyConfigRouter()
+	router, store := getDummyConfigRouter(t)
 
 	storedcfg := store.Get()
 
@@ -57,7 +62,7 @@ func TestConfigSet(t *testing.T) {
 	encoder := json.NewEncoder(&data)
 
 	// Setting a new v3 config
-	cfg := config.New()
+	cfg := config.New(nil)
 	cfg.FFmpeg.Binary = "true"
 	cfg.DB.Dir = "."
 	cfg.Storage.Disk.Dir = "."
@@ -78,7 +83,7 @@ func TestConfigSet(t *testing.T) {
 	require.Equal(t, "cert@datarhei.com", cfg.TLS.Email)
 
 	// Setting a complete v1 config
-	cfgv1 := v1.New()
+	cfgv1 := v1.New(nil)
 	cfgv1.FFmpeg.Binary = "true"
 	cfgv1.DB.Dir = "."
 	cfgv1.Storage.Disk.Dir = "."

http/handler/filesystem.go

@@ -82,7 +82,7 @@ func (h *FSHandler) PutFile(c echo.Context) error {
 
 	req := c.Request()
 
-	_, created, err := h.fs.Filesystem.Store(path, req.Body)
+	_, created, err := h.fs.Filesystem.WriteFileReader(path, req.Body)
 	if err != nil {
 		return api.Err(http.StatusBadRequest, "%s", err)
 	}
@@ -105,7 +105,7 @@ func (h *FSHandler) DeleteFile(c echo.Context) error {
 
 	c.Response().Header().Del(echo.HeaderContentType)
 
-	size := h.fs.Filesystem.Delete(path)
+	size := h.fs.Filesystem.Remove(path)
 
 	if size < 0 {
 		return api.Err(http.StatusNotFound, "File not found", path)
@@ -123,7 +123,7 @@ func (h *FSHandler) ListFiles(c echo.Context) error {
 	sortby := util.DefaultQuery(c, "sort", "none")
 	order := util.DefaultQuery(c, "order", "asc")
 
-	files := h.fs.Filesystem.List(pattern)
+	files := h.fs.Filesystem.List("/", pattern)
 
 	var sortFunc func(i, j int) bool
 

http/mock/mock.go

@@ -17,6 +17,7 @@ import (
 	"github.com/datarhei/core/v16/http/errorhandler"
 	"github.com/datarhei/core/v16/http/validator"
 	"github.com/datarhei/core/v16/internal/testhelper"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/restream"
 	"github.com/datarhei/core/v16/restream/store"
 
@@ -32,7 +33,17 @@ func DummyRestreamer(pathPrefix string) (restream.Restreamer, error) {
 		return nil, fmt.Errorf("failed to build helper program: %w", err)
 	}
 
-	store := store.NewDummyStore(store.DummyConfig{})
+	memfs, err := fs.NewMemFilesystem(fs.MemConfig{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create memory filesystem: %w", err)
+	}
+
+	store, err := store.NewJSON(store.JSONConfig{
+		Filesystem: memfs,
+	})
+	if err != nil {
+		return nil, err
+	}
 
 	ffmpeg, err := ffmpeg.New(ffmpeg.Config{
 		Binary: binary,

http/server.go

@@ -198,7 +198,7 @@ func NewServer(config Config) (Server, error) {
 
 		if fs.Filesystem.Type() == "disk" {
 			s.middleware.hlsrewrite = mwhlsrewrite.NewHLSRewriteWithConfig(mwhlsrewrite.HLSRewriteConfig{
-				PathPrefix: fs.Filesystem.Base(),
+				PathPrefix: fs.Filesystem.Metadata("base"),
 			})
 		}
 	}

io/fs/disk.go

@@ -1,28 +1,30 @@
 package fs
 
 import (
+	"bytes"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/datarhei/core/v16/glob"
 	"github.com/datarhei/core/v16/log"
 )
 
-// DiskConfig is the config required to create a new disk
-// filesystem.
+// DiskConfig is the config required to create a new disk filesystem.
 type DiskConfig struct {
-	// Namee is the name of the filesystem
-	Name string
+	// For logging, optional
+	Logger log.Logger
+}
 
-	// Dir is the path to the directory to observe
-	Dir string
-
-	// Size of the filesystem in bytes
-	Size int64
+// RootedDiskConfig is the config required to create a new rooted disk filesystem.
+type RootedDiskConfig struct {
+	// Root is the path this filesystem is rooted to
+	Root string
 
 	// For logging, optional
 	Logger log.Logger
@@ -30,8 +32,9 @@ type DiskConfig struct {
 
 // diskFileInfo implements the FileInfo interface
 type diskFileInfo struct {
-	dir   string
+	root  string
 	name  string
+	mode  os.FileMode
 	finfo os.FileInfo
 }
 
@@ -40,31 +43,37 @@ func (fi *diskFileInfo) Name() string {
 }
 
 func (fi *diskFileInfo) Size() int64 {
+	if fi.finfo.IsDir() {
+		return 0
+	}
+
 	return fi.finfo.Size()
 }
 
+func (fi *diskFileInfo) Mode() fs.FileMode {
+	return fi.mode
+}
+
 func (fi *diskFileInfo) ModTime() time.Time {
 	return fi.finfo.ModTime()
 }
 
 func (fi *diskFileInfo) IsLink() (string, bool) {
-	mode := fi.finfo.Mode()
-	if mode&os.ModeSymlink == 0 {
+	if fi.mode&os.ModeSymlink == 0 {
 		return fi.name, false
 	}
 
-	path, err := os.Readlink(filepath.Join(fi.dir, fi.name))
+	path, err := os.Readlink(filepath.Join(fi.root, fi.name))
 	if err != nil {
 		return fi.name, false
 	}
 
-	path = filepath.Join(fi.dir, path)
-
-	if !strings.HasPrefix(path, fi.dir) {
+	if !strings.HasPrefix(path, fi.root) {
 		return fi.name, false
 	}
 
-	name := strings.TrimPrefix(path, fi.dir)
+	name := strings.TrimPrefix(path, fi.root)
+
 	if name[0] != os.PathSeparator {
 		name = string(os.PathSeparator) + name
 	}
@@ -78,8 +87,9 @@ func (fi *diskFileInfo) IsDir() bool {
 
 // diskFile implements the File interface
 type diskFile struct {
-	dir  string
+	root string
 	name string
+	mode os.FileMode
 	file *os.File
 }
 
@@ -94,8 +104,9 @@ func (f *diskFile) Stat() (FileInfo, error) {
 	}
 
 	dif := &diskFileInfo{
-		dir:   f.dir,
+		root:  f.root,
 		name:  f.name,
+		mode:  f.mode,
 		finfo: finfo,
 	}
 
@@ -112,12 +123,11 @@ func (f *diskFile) Read(p []byte) (int, error) {
 
 // diskFilesystem implements the Filesystem interface
 type diskFilesystem struct {
-	name string
-	dir  string
+	metadata map[string]string
+	lock     sync.RWMutex
 
-	// Max. size of the filesystem in bytes as
-	// given by the config
-	maxSize int64
+	root string
+	cwd  string
 
 	// Current size of the filesystem in bytes
 	currentSize   int64
@@ -127,67 +137,102 @@ type diskFilesystem struct {
 	logger log.Logger
 }
 
-// NewDiskFilesystem returns a new filesystem that is backed by a disk
-// that implements the Filesystem interface
+// NewDiskFilesystem returns a new filesystem that is backed by the disk filesystem.
+// The root is / and the working directory is whatever is returned by os.Getwd(). The value
+// of Root in the config will be ignored.
 func NewDiskFilesystem(config DiskConfig) (Filesystem, error) {
 	fs := &diskFilesystem{
-		name:    config.Name,
-		maxSize: config.Size,
-		logger:  config.Logger,
+		metadata: make(map[string]string),
+		root:     "/",
+		cwd:      "/",
+		logger:   config.Logger,
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+
+	fs.cwd = cwd
+
+	if len(fs.cwd) == 0 {
+		fs.cwd = "/"
+	}
+
+	fs.cwd = filepath.Clean(fs.cwd)
+	if !filepath.IsAbs(fs.cwd) {
+		return nil, fmt.Errorf("the current working directory must be an absolute path")
 	}
 
 	if fs.logger == nil {
 		fs.logger = log.New("")
 	}
 
-	fs.logger = fs.logger.WithFields(log.Fields{
-		"name": fs.name,
-		"type": "disk",
-	})
+	return fs, nil
+}
 
-	if err := fs.Rebase(config.Dir); err != nil {
+// NewRootedDiskFilesystem returns a filesystem that is backed by the disk filesystem. The
+// root of the filesystem is defined by DiskConfig.Root. The working directory is "/". Root
+// must be directory. If it doesn't exist, it will be created
+func NewRootedDiskFilesystem(config RootedDiskConfig) (Filesystem, error) {
+	fs := &diskFilesystem{
+		metadata: make(map[string]string),
+		root:     config.Root,
+		cwd:      "/",
+		logger:   config.Logger,
+	}
+
+	if len(fs.root) == 0 {
+		fs.root = "/"
+	}
+
+	if root, err := filepath.Abs(fs.root); err != nil {
 		return nil, err
+	} else {
+		fs.root = root
+	}
+
+	err := os.MkdirAll(fs.root, 0700)
+	if err != nil {
+		return nil, err
+	}
+
+	info, err := os.Stat(fs.root)
+	if err != nil {
+		return nil, err
+	}
+
+	if !info.IsDir() {
+		return nil, fmt.Errorf("root is not a directory")
+	}
+
+	if fs.logger == nil {
+		fs.logger = log.New("")
 	}
 
 	return fs, nil
 }
 
 func (fs *diskFilesystem) Name() string {
-	return fs.name
-}
-
-func (fs *diskFilesystem) Base() string {
-	return fs.dir
-}
-
-func (fs *diskFilesystem) Rebase(base string) error {
-	if len(base) == 0 {
-		return fmt.Errorf("invalid base path provided")
-	}
-
-	dir, err := filepath.Abs(base)
-	if err != nil {
-		return err
-	}
-
-	base = dir
-
-	finfo, err := os.Stat(base)
-	if err != nil {
-		return fmt.Errorf("the provided base path '%s' doesn't exist", fs.dir)
-	}
-
-	if !finfo.IsDir() {
-		return fmt.Errorf("the provided base path '%s' must be a directory", fs.dir)
-	}
-
-	fs.dir = base
-
-	return nil
+	return "disk"
 }
 
 func (fs *diskFilesystem) Type() string {
-	return "diskfs"
+	return "disk"
+}
+
+func (fs *diskFilesystem) Metadata(key string) string {
+	fs.lock.RLock()
+	defer fs.lock.RUnlock()
+
+	return fs.metadata[key]
+}
+
+func (fs *diskFilesystem) SetMetadata(key, data string) {
+	fs.lock.Lock()
+	defer fs.lock.Unlock()
+
+	fs.metadata[key] = data
 }
 
 func (fs *diskFilesystem) Size() (int64, int64) {
@@ -196,7 +241,11 @@ func (fs *diskFilesystem) Size() (int64, int64) {
 	if time.Since(fs.lastSizeCheck) >= 10*time.Second {
 		var size int64 = 0
 
-		fs.walk(func(path string, info os.FileInfo) {
+		fs.walk(fs.root, func(path string, info os.FileInfo) {
+			if info.IsDir() {
+				return
+			}
+
 			size += info.Size()
 		})
 
@@ -205,17 +254,21 @@ func (fs *diskFilesystem) Size() (int64, int64) {
 		fs.lastSizeCheck = time.Now()
 	}
 
-	return fs.currentSize, fs.maxSize
+	return fs.currentSize, -1
 }
 
-func (fs *diskFilesystem) Resize(size int64) {
-	fs.maxSize = size
+func (fs *diskFilesystem) Purge(size int64) int64 {
+	return 0
 }
 
 func (fs *diskFilesystem) Files() int64 {
 	var nfiles int64 = 0
 
-	fs.walk(func(path string, info os.FileInfo) {
+	fs.walk(fs.root, func(path string, info os.FileInfo) {
+		if info.IsDir() {
+			return
+		}
+
 		nfiles++
 	})
 
@@ -223,38 +276,58 @@ func (fs *diskFilesystem) Files() int64 {
 }
 
 func (fs *diskFilesystem) Symlink(oldname, newname string) error {
-	oldname = filepath.Join(fs.dir, filepath.Clean("/"+oldname))
+	oldname = fs.cleanPath(oldname)
+	newname = fs.cleanPath(newname)
 
-	if !filepath.IsAbs(newname) {
-		return nil
+	info, err := os.Lstat(oldname)
+	if err != nil {
+		return err
 	}
 
-	newname = filepath.Join(fs.dir, filepath.Clean("/"+newname))
+	if info.Mode()&os.ModeSymlink != 0 {
+		return fmt.Errorf("%s can't link to another link (%s)", newname, oldname)
+	}
 
-	err := os.Symlink(oldname, newname)
+	if info.IsDir() {
+		return fmt.Errorf("can't symlink directories")
+	}
 
-	return err
+	return os.Symlink(oldname, newname)
 }
 
 func (fs *diskFilesystem) Open(path string) File {
-	path = filepath.Join(fs.dir, filepath.Clean("/"+path))
+	path = fs.cleanPath(path)
+
+	df := &diskFile{
+		root: fs.root,
+		name: strings.TrimPrefix(path, fs.root),
+	}
+
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil
+	}
+
+	df.mode = info.Mode()
 
 	f, err := os.Open(path)
 	if err != nil {
 		return nil
 	}
 
-	df := &diskFile{
-		dir:  fs.dir,
-		name: path,
-		file: f,
-	}
+	df.file = f
 
 	return df
 }
 
-func (fs *diskFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
-	path = filepath.Join(fs.dir, filepath.Clean("/"+path))
+func (fs *diskFilesystem) ReadFile(path string) ([]byte, error) {
+	path = fs.cleanPath(path)
+
+	return os.ReadFile(path)
+}
+
+func (fs *diskFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
+	path = fs.cleanPath(path)
 
 	replace := true
 
@@ -276,16 +349,155 @@ func (fs *diskFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
 		replace = false
 	}
 
+	defer f.Close()
+
 	size, err := f.ReadFrom(r)
 	if err != nil {
 		return -1, false, fmt.Errorf("reading data failed: %w", err)
 	}
 
+	fs.lastSizeCheck = time.Time{}
+
 	return size, !replace, nil
 }
 
-func (fs *diskFilesystem) Delete(path string) int64 {
-	path = filepath.Join(fs.dir, filepath.Clean("/"+path))
+func (fs *diskFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
+	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (fs *diskFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
+	path = fs.cleanPath(path)
+	dir, filename := filepath.Split(path)
+
+	tmpfile, err := os.CreateTemp(dir, filename)
+	if err != nil {
+		return -1, false, err
+	}
+
+	defer os.Remove(tmpfile.Name())
+
+	size, err := tmpfile.Write(data)
+	if err != nil {
+		return -1, false, err
+	}
+
+	if err := tmpfile.Close(); err != nil {
+		return -1, false, err
+	}
+
+	replace := false
+	if _, err := fs.Stat(path); err == nil {
+		replace = true
+	}
+
+	if err := fs.rename(tmpfile.Name(), path); err != nil {
+		return -1, false, err
+	}
+
+	fs.lastSizeCheck = time.Time{}
+
+	return int64(size), !replace, nil
+}
+
+func (fs *diskFilesystem) Rename(src, dst string) error {
+	src = fs.cleanPath(src)
+	dst = fs.cleanPath(dst)
+
+	return fs.rename(src, dst)
+}
+
+func (fs *diskFilesystem) rename(src, dst string) error {
+	if src == dst {
+		return nil
+	}
+
+	// First try to rename the file
+	if err := os.Rename(src, dst); err == nil {
+		return nil
+	}
+
+	// If renaming the file fails, copy the data
+	if err := fs.copy(src, dst); err != nil {
+		os.Remove(dst)
+		return fmt.Errorf("failed to copy files: %w", err)
+	}
+
+	if err := os.Remove(src); err != nil {
+		os.Remove(dst)
+		return fmt.Errorf("failed to remove source file: %w", err)
+	}
+
+	return nil
+}
+
+func (fs *diskFilesystem) Copy(src, dst string) error {
+	src = fs.cleanPath(src)
+	dst = fs.cleanPath(dst)
+
+	return fs.copy(src, dst)
+}
+
+func (fs *diskFilesystem) copy(src, dst string) error {
+	source, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open source file: %w", err)
+	}
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		source.Close()
+		return fmt.Errorf("failed to create destination file: %w", err)
+	}
+	defer destination.Close()
+
+	if _, err := io.Copy(destination, source); err != nil {
+		source.Close()
+		os.Remove(dst)
+		return fmt.Errorf("failed to copy data from source to destination: %w", err)
+	}
+
+	source.Close()
+
+	fs.lastSizeCheck = time.Time{}
+
+	return nil
+}
+
+func (fs *diskFilesystem) MkdirAll(path string, perm os.FileMode) error {
+	path = fs.cleanPath(path)
+
+	return os.MkdirAll(path, perm)
+}
+
+func (fs *diskFilesystem) Stat(path string) (FileInfo, error) {
+	path = fs.cleanPath(path)
+
+	dif := &diskFileInfo{
+		root: fs.root,
+		name: strings.TrimPrefix(path, fs.root),
+	}
+
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	dif.mode = info.Mode()
+
+	if info.Mode()&os.ModeSymlink != 0 {
+		info, err = os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	dif.finfo = info
+
+	return dif, nil
+}
+
+func (fs *diskFilesystem) Remove(path string) int64 {
+	path = fs.cleanPath(path)
 
 	finfo, err := os.Stat(path)
 	if err != nil {
@@ -298,28 +510,31 @@ func (fs *diskFilesystem) Delete(path string) int64 {
 		return -1
 	}
 
+	fs.lastSizeCheck = time.Time{}
+
 	return size
 }
 
-func (fs *diskFilesystem) DeleteAll() int64 {
+func (fs *diskFilesystem) RemoveAll() int64 {
 	return 0
 }
 
-func (fs *diskFilesystem) List(pattern string) []FileInfo {
+func (fs *diskFilesystem) List(path, pattern string) []FileInfo {
+	path = fs.cleanPath(path)
 	files := []FileInfo{}
 
-	fs.walk(func(path string, info os.FileInfo) {
-		if path == fs.dir {
+	fs.walk(path, func(path string, info os.FileInfo) {
+		if path == fs.root {
 			return
 		}
 
-		name := strings.TrimPrefix(path, fs.dir)
+		name := strings.TrimPrefix(path, fs.root)
 		if name[0] != os.PathSeparator {
 			name = string(os.PathSeparator) + name
 		}
 
 		if info.IsDir() {
-			name += "/"
+			return
 		}
 
 		if len(pattern) != 0 {
@@ -329,7 +544,7 @@ func (fs *diskFilesystem) List(pattern string) []FileInfo {
 		}
 
 		files = append(files, &diskFileInfo{
-			dir:   fs.dir,
+			root:  fs.root,
 			name:  name,
 			finfo: info,
 		})
@@ -338,8 +553,8 @@ func (fs *diskFilesystem) List(pattern string) []FileInfo {
 	return files
 }
 
-func (fs *diskFilesystem) walk(walkfn func(path string, info os.FileInfo)) {
-	filepath.Walk(fs.dir, func(path string, info os.FileInfo, err error) error {
+func (fs *diskFilesystem) walk(path string, walkfn func(path string, info os.FileInfo)) {
+	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return nil
 		}
@@ -359,3 +574,11 @@ func (fs *diskFilesystem) walk(walkfn func(path string, info os.FileInfo)) {
 		return nil
 	})
 }
+
+func (fs *diskFilesystem) cleanPath(path string) string {
+	if !filepath.IsAbs(path) {
+		path = filepath.Join(fs.cwd, path)
+	}
+
+	return filepath.Join(fs.root, filepath.Clean(path))
+}

io/fs/dummy.go

@@ -1,48 +0,0 @@
-package fs
-
-import (
-	"io"
-	"time"
-)
-
-type dummyFileInfo struct{}
-
-func (d *dummyFileInfo) Name() string           { return "" }
-func (d *dummyFileInfo) Size() int64            { return 0 }
-func (d *dummyFileInfo) ModTime() time.Time     { return time.Date(2000, 1, 1, 0, 0, 0, 0, nil) }
-func (d *dummyFileInfo) IsLink() (string, bool) { return "", false }
-func (d *dummyFileInfo) IsDir() bool            { return false }
-
-type dummyFile struct{}
-
-func (d *dummyFile) Read(p []byte) (int, error) { return 0, io.EOF }
-func (d *dummyFile) Close() error               { return nil }
-func (d *dummyFile) Name() string               { return "" }
-func (d *dummyFile) Stat() (FileInfo, error)    { return &dummyFileInfo{}, nil }
-
-type dummyFilesystem struct {
-	name string
-	typ  string
-}
-
-func (d *dummyFilesystem) Name() string                                 { return d.name }
-func (d *dummyFilesystem) Base() string                                 { return "/" }
-func (d *dummyFilesystem) Rebase(string) error                          { return nil }
-func (d *dummyFilesystem) Type() string                                 { return d.typ }
-func (d *dummyFilesystem) Size() (int64, int64)                         { return 0, -1 }
-func (d *dummyFilesystem) Resize(int64)                                 {}
-func (d *dummyFilesystem) Files() int64                                 { return 0 }
-func (d *dummyFilesystem) Symlink(string, string) error                 { return nil }
-func (d *dummyFilesystem) Open(string) File                             { return &dummyFile{} }
-func (d *dummyFilesystem) Store(string, io.Reader) (int64, bool, error) { return 0, true, nil }
-func (d *dummyFilesystem) Delete(string) int64                          { return 0 }
-func (d *dummyFilesystem) DeleteAll() int64                             { return 0 }
-func (d *dummyFilesystem) List(string) []FileInfo                       { return []FileInfo{} }
-
-// NewDummyFilesystem return a dummy filesystem
-func NewDummyFilesystem(name, typ string) Filesystem {
-	return &dummyFilesystem{
-		name: name,
-		typ:  typ,
-	}
-}

io/fs/fs.go

@@ -3,24 +3,29 @@ package fs
 
 import (
 	"io"
+	"io/fs"
+	"os"
 	"time"
 )
 
 // FileInfo describes a file and is returned by Stat.
 type FileInfo interface {
-	// Name returns the full name of the file
+	// Name returns the full name of the file.
 	Name() string
 
-	// Size reports the size of the file in bytes
+	// Size reports the size of the file in bytes.
 	Size() int64
 
-	// ModTime returns the time of last modification
+	// Mode returns the file mode.
+	Mode() fs.FileMode
+
+	// ModTime returns the time of last modification.
 	ModTime() time.Time
 
 	// IsLink returns the path this file is linking to and true. Otherwise an empty string and false.
 	IsLink() (string, bool)
 
-	// IsDir returns whether the file represents a directory
+	// IsDir returns whether the file represents a directory.
 	IsDir() bool
 }
 
@@ -28,58 +33,95 @@ type FileInfo interface {
 type File interface {
 	io.ReadCloser
 
-	// Name returns the Name of the file
+	// Name returns the Name of the file.
 	Name() string
 
-	// Stat returns the FileInfo to this file. In case of an error
-	// FileInfo is nil and the error is non-nil.
+	// Stat returns the FileInfo to this file. In case of an error FileInfo is nil
+	// and the error is non-nil. If the file is a symlink, the info reports the name and mode
+	// of the link itself, but the modification time and size of the linked file.
 	Stat() (FileInfo, error)
 }
 
-// Filesystem is an interface that provides access to a filesystem.
-type Filesystem interface {
-	// Name returns the name of this filesystem
-	Name() string
-
-	// Base returns the base path of this filesystem
-	Base() string
-
-	// Rebase sets a new base path for this filesystem
-	Rebase(string) error
-
-	// Type returns the type of this filesystem
-	Type() string
-
+type ReadFilesystem interface {
 	// Size returns the consumed size and capacity of the filesystem in bytes. The
-	// capacity is negative if the filesystem can consume as much space as it can.
+	// capacity is negative if the filesystem can consume as much space as it wants.
 	Size() (int64, int64)
 
-	// Resize resizes the filesystem to the new size. Files may need to be deleted.
-	Resize(size int64)
-
 	// Files returns the current number of files in the filesystem.
 	Files() int64
 
+	// Open returns the file stored at the given path. It returns nil if the
+	// file doesn't exist. If the file is a symlink, the name is the name of
+	// the link, but it will read the contents of the linked file.
+	Open(path string) File
+
+	// ReadFile reads the content of the file at the given path into the writer. Returns
+	// the number of bytes read or an error.
+	ReadFile(path string) ([]byte, error)
+
+	// Stat returns info about the file at path. If the file doesn't exist, an error
+	// will be returned. If the file is a symlink, the info reports the name and mode
+	// of the link itself, but the modification time and size are of the linked file.
+	Stat(path string) (FileInfo, error)
+
+	// List lists all files that are currently on the filesystem.
+	List(path, pattern string) []FileInfo
+}
+
+type WriteFilesystem interface {
 	// Symlink creates newname as a symbolic link to oldname.
 	Symlink(oldname, newname string) error
 
-	// Open returns the file stored at the given path. It returns nil if the
-	// file doesn't exist.
-	Open(path string) File
-
-	// Store adds a file to the filesystem. Returns the size of the data that has been
+	// WriteFileReader adds a file to the filesystem. Returns the size of the data that has been
 	// stored in bytes and whether the file is new. The size is negative if there was
 	// an error adding the file and error is not nil.
-	Store(path string, r io.Reader) (int64, bool, error)
+	WriteFileReader(path string, r io.Reader) (int64, bool, error)
 
-	// Delete removes a file at the given path from the filesystem. Returns the size of
-	// the removed file in bytes. The size is negative if the file doesn't exist.
-	Delete(path string) int64
+	// WriteFile adds a file to the filesystem. Returns the size of the data that has been
+	// stored in bytes and whether the file is new. The size is negative if there was
+	// an error adding the file and error is not nil.
+	WriteFile(path string, data []byte) (int64, bool, error)
 
-	// DeleteAll removes all files from the filesystem. Returns the size of the
+	// WriteFileSafe adds a file to the filesystem by first writing it to a tempfile and then
+	// renaming it to the actual path. Returns the size of the data that has been
+	// stored in bytes and whether the file is new. The size is negative if there was
+	// an error adding the file and error is not nil.
+	WriteFileSafe(path string, data []byte) (int64, bool, error)
+
+	// MkdirAll creates a directory named path, along with any necessary parents, and returns nil,
+	// or else returns an error. The permission bits perm (before umask) are used for all directories
+	// that MkdirAll creates. If path is already a directory, MkdirAll does nothing and returns nil.
+	MkdirAll(path string, perm os.FileMode) error
+
+	// Rename renames the file from src to dst. If src and dst can't be renamed
+	// regularly, the data is copied from src to dst. dst will be overwritten
+	// if it already exists. src will be removed after all data has been copied
+	// successfully. Both files exist during copying.
+	Rename(src, dst string) error
+
+	// Copy copies a file from src to dst.
+	Copy(src, dst string) error
+
+	// Remove removes a file at the given path from the filesystem. Returns the size of
+	// the remove file in bytes. The size is negative if the file doesn't exist.
+	Remove(path string) int64
+
+	// RemoveAll removes all files from the filesystem. Returns the size of the
 	// removed files in bytes.
-	DeleteAll() int64
-
-	// List lists all files that are currently on the filesystem.
-	List(pattern string) []FileInfo
+	RemoveAll() int64
+}
+
+// Filesystem is an interface that provides access to a filesystem.
+type Filesystem interface {
+	ReadFilesystem
+	WriteFilesystem
+
+	// Name returns the name of the filesystem.
+	Name() string
+
+	// Type returns the type of the filesystem, e.g. disk, mem, s3
+	Type() string
+
+	Metadata(key string) string
+	SetMetadata(key string, data string)
 }

io/fs/fs_test.go

@@ -0,0 +1,742 @@
+package fs
+
+import (
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var ErrNoMinio = errors.New("minio binary not found")
+
+func startMinio(t *testing.T, path string) (*exec.Cmd, error) {
+	err := os.MkdirAll(path, 0700)
+	require.NoError(t, err)
+
+	minio, err := exec.LookPath("minio")
+	if err != nil {
+		return nil, ErrNoMinio
+	}
+
+	proc := exec.Command(minio, "server", path, "--address", "127.0.0.1:9000")
+	proc.Stderr = os.Stderr
+	proc.Stdout = os.Stdout
+	err = proc.Start()
+	require.NoError(t, err)
+
+	time.Sleep(5 * time.Second)
+
+	return proc, nil
+}
+
+func stopMinio(t *testing.T, proc *exec.Cmd) {
+	err := proc.Process.Signal(os.Interrupt)
+	require.NoError(t, err)
+
+	proc.Wait()
+}
+
+func TestFilesystem(t *testing.T) {
+	miniopath, err := filepath.Abs("./minio")
+	require.NoError(t, err)
+
+	err = os.RemoveAll(miniopath)
+	require.NoError(t, err)
+
+	minio, err := startMinio(t, miniopath)
+	if err != nil {
+		if err != ErrNoMinio {
+			require.NoError(t, err)
+		}
+	}
+
+	os.RemoveAll("./testing/")
+
+	filesystems := map[string]func(string) (Filesystem, error){
+		"memfs": func(name string) (Filesystem, error) {
+			return NewMemFilesystem(MemConfig{})
+		},
+		"diskfs": func(name string) (Filesystem, error) {
+			return NewRootedDiskFilesystem(RootedDiskConfig{
+				Root: "./testing/" + name,
+			})
+		},
+		"s3fs": func(name string) (Filesystem, error) {
+			return NewS3Filesystem(S3Config{
+				Name:            name,
+				Endpoint:        "127.0.0.1:9000",
+				AccessKeyID:     "minioadmin",
+				SecretAccessKey: "minioadmin",
+				Region:          "",
+				Bucket:          strings.ToLower(name),
+				UseSSL:          false,
+				Logger:          nil,
+			})
+		},
+	}
+
+	tests := map[string]func(*testing.T, Filesystem){
+		"new":             testNew,
+		"metadata":        testMetadata,
+		"writeFile":       testWriteFile,
+		"writeFileSafe":   testWriteFileSafe,
+		"writeFileReader": testWriteFileReader,
+		"delete":          testDelete,
+		"files":           testFiles,
+		"replace":         testReplace,
+		"list":            testList,
+		"listGlob":        testListGlob,
+		"deleteAll":       testDeleteAll,
+		"data":            testData,
+		"statDir":         testStatDir,
+		"mkdirAll":        testMkdirAll,
+		"rename":          testRename,
+		"renameOverwrite": testRenameOverwrite,
+		"copy":            testCopy,
+		"symlink":         testSymlink,
+		"stat":            testStat,
+		"copyOverwrite":   testCopyOverwrite,
+		"symlinkErrors":   testSymlinkErrors,
+		"symlinkOpenStat": testSymlinkOpenStat,
+		"open":            testOpen,
+	}
+
+	for fsname, fs := range filesystems {
+		for name, test := range tests {
+			t.Run(fsname+"-"+name, func(t *testing.T) {
+				if fsname == "s3fs" && minio == nil {
+					t.Skip("minio server not available")
+				}
+				filesystem, err := fs(name)
+				require.NoError(t, err)
+				test(t, filesystem)
+			})
+		}
+	}
+
+	os.RemoveAll("./testing/")
+
+	if minio != nil {
+		stopMinio(t, minio)
+	}
+
+	os.RemoveAll(miniopath)
+}
+
+func testNew(t *testing.T, fs Filesystem) {
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur, "current size")
+	require.Equal(t, int64(-1), max, "max size")
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(0), cur, "number of files")
+}
+
+func testMetadata(t *testing.T, fs Filesystem) {
+	fs.SetMetadata("foo", "bar")
+	require.Equal(t, "bar", fs.Metadata("foo"))
+}
+
+func testWriteFile(t *testing.T, fs Filesystem) {
+	size, created, err := fs.WriteFile("/foobar", []byte("xxxxx"))
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func testWriteFileSafe(t *testing.T, fs Filesystem) {
+	size, created, err := fs.WriteFileSafe("/foobar", []byte("xxxxx"))
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func testWriteFileReader(t *testing.T, fs Filesystem) {
+	data := strings.NewReader("xxxxx")
+
+	size, created, err := fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func testOpen(t *testing.T, fs Filesystem) {
+	file := fs.Open("/foobar")
+	require.Nil(t, file)
+
+	_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("xxxxx"))
+	require.NoError(t, err)
+
+	file = fs.Open("/foobar")
+	require.NotNil(t, file)
+	require.Equal(t, "/foobar", file.Name())
+
+	stat, err := file.Stat()
+	require.NoError(t, err)
+	require.Equal(t, "/foobar", stat.Name())
+	require.Equal(t, int64(5), stat.Size())
+	require.Equal(t, false, stat.IsDir())
+}
+
+func testDelete(t *testing.T, fs Filesystem) {
+	size := fs.Remove("/foobar")
+
+	require.Equal(t, int64(-1), size)
+
+	data := strings.NewReader("xxxxx")
+
+	fs.WriteFileReader("/foobar", data)
+
+	size = fs.Remove("/foobar")
+
+	require.Equal(t, int64(5), size)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(0), cur)
+}
+
+func testFiles(t *testing.T, fs Filesystem) {
+	require.Equal(t, int64(0), fs.Files())
+
+	fs.WriteFileReader("/foobar.txt", strings.NewReader("bar"))
+
+	require.Equal(t, int64(1), fs.Files())
+
+	fs.MkdirAll("/path/to/foo", 0777)
+
+	require.Equal(t, int64(1), fs.Files())
+
+	fs.Remove("/foobar.txt")
+
+	require.Equal(t, int64(0), fs.Files())
+}
+
+func testReplace(t *testing.T, fs Filesystem) {
+	data := strings.NewReader("xxxxx")
+
+	size, created, err := fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+
+	data = strings.NewReader("yyy")
+
+	size, created, err = fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(3), size)
+	require.Equal(t, false, created)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(3), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func testList(t *testing.T, fs Filesystem) {
+	fs.WriteFileReader("/foobar1", strings.NewReader("a"))
+	fs.WriteFileReader("/foobar2", strings.NewReader("bb"))
+	fs.WriteFileReader("/foobar3", strings.NewReader("ccc"))
+	fs.WriteFileReader("/foobar4", strings.NewReader("dddd"))
+	fs.WriteFileReader("/path/foobar3", strings.NewReader("ccc"))
+	fs.WriteFileReader("/path/to/foobar4", strings.NewReader("dddd"))
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(17), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(6), cur)
+
+	getNames := func(files []FileInfo) []string {
+		names := []string{}
+		for _, f := range files {
+			names = append(names, f.Name())
+		}
+		return names
+	}
+
+	files := fs.List("/", "")
+
+	require.Equal(t, 6, len(files))
+	require.ElementsMatch(t, []string{"/foobar1", "/foobar2", "/foobar3", "/foobar4", "/path/foobar3", "/path/to/foobar4"}, getNames(files))
+
+	files = fs.List("/path", "")
+
+	require.Equal(t, 2, len(files))
+	require.ElementsMatch(t, []string{"/path/foobar3", "/path/to/foobar4"}, getNames(files))
+}
+
+func testListGlob(t *testing.T, fs Filesystem) {
+	fs.WriteFileReader("/foobar1", strings.NewReader("a"))
+	fs.WriteFileReader("/path/foobar2", strings.NewReader("a"))
+	fs.WriteFileReader("/path/to/foobar3", strings.NewReader("a"))
+	fs.WriteFileReader("/foobar4", strings.NewReader("a"))
+
+	cur := fs.Files()
+
+	require.Equal(t, int64(4), cur)
+
+	getNames := func(files []FileInfo) []string {
+		names := []string{}
+		for _, f := range files {
+			names = append(names, f.Name())
+		}
+		return names
+	}
+
+	files := getNames(fs.List("/", "/foo*"))
+	require.Equal(t, 2, len(files))
+	require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
+
+	files = getNames(fs.List("/", "/*bar?"))
+	require.Equal(t, 2, len(files))
+	require.ElementsMatch(t, []string{"/foobar1", "/foobar4"}, files)
+
+	files = getNames(fs.List("/", "/path/*"))
+	require.Equal(t, 1, len(files))
+	require.ElementsMatch(t, []string{"/path/foobar2"}, files)
+
+	files = getNames(fs.List("/", "/path/**"))
+	require.Equal(t, 2, len(files))
+	require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
+
+	files = getNames(fs.List("/path", "/**"))
+	require.Equal(t, 2, len(files))
+	require.ElementsMatch(t, []string{"/path/foobar2", "/path/to/foobar3"}, files)
+}
+
+func testDeleteAll(t *testing.T, fs Filesystem) {
+	if _, ok := fs.(*diskFilesystem); ok {
+		return
+	}
+
+	fs.WriteFileReader("/foobar1", strings.NewReader("abc"))
+	fs.WriteFileReader("/path/foobar2", strings.NewReader("abc"))
+	fs.WriteFileReader("/path/to/foobar3", strings.NewReader("abc"))
+	fs.WriteFileReader("/foobar4", strings.NewReader("abc"))
+
+	cur := fs.Files()
+
+	require.Equal(t, int64(4), cur)
+
+	size := fs.RemoveAll()
+	require.Equal(t, int64(12), size)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(0), cur)
+}
+
+func testData(t *testing.T, fs Filesystem) {
+	file := fs.Open("/foobar")
+	require.Nil(t, file)
+
+	_, err := fs.ReadFile("/foobar")
+	require.Error(t, err)
+
+	data := "gduwotoxqb"
+
+	data1 := strings.NewReader(data)
+
+	_, _, err = fs.WriteFileReader("/foobar", data1)
+	require.NoError(t, err)
+
+	file = fs.Open("/foobar")
+	require.NotNil(t, file)
+
+	data2 := make([]byte, len(data)+1)
+	n, err := file.Read(data2)
+	if err != nil {
+		if err != io.EOF {
+			require.NoError(t, err)
+		}
+	}
+
+	require.Equal(t, len(data), n)
+	require.Equal(t, []byte(data), data2[:n])
+
+	data3, err := fs.ReadFile("/foobar")
+
+	require.NoError(t, err)
+	require.Equal(t, []byte(data), data3)
+}
+
+func testStatDir(t *testing.T, fs Filesystem) {
+	info, err := fs.Stat("/")
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	require.Equal(t, true, info.IsDir())
+
+	data := strings.NewReader("gduwotoxqb")
+	fs.WriteFileReader("/these/are/some/directories/foobar", data)
+
+	info, err = fs.Stat("/foobar")
+	require.Error(t, err)
+	require.Nil(t, info)
+
+	info, err = fs.Stat("/these/are/some/directories/foobar")
+	require.NoError(t, err)
+	require.Equal(t, "/these/are/some/directories/foobar", info.Name())
+	require.Equal(t, int64(10), info.Size())
+	require.Equal(t, false, info.IsDir())
+
+	info, err = fs.Stat("/these")
+	require.NoError(t, err)
+	require.Equal(t, "/these", info.Name())
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/these/are/")
+	require.NoError(t, err)
+	require.Equal(t, "/these/are", info.Name())
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/these/are/some")
+	require.NoError(t, err)
+	require.Equal(t, "/these/are/some", info.Name())
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/these/are/some/directories")
+	require.NoError(t, err)
+	require.Equal(t, "/these/are/some/directories", info.Name())
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+}
+
+func testMkdirAll(t *testing.T, fs Filesystem) {
+	info, err := fs.Stat("/foo/bar/dir")
+	require.Error(t, err)
+	require.Nil(t, info)
+
+	err = fs.MkdirAll("/foo/bar/dir", 0755)
+	require.NoError(t, err)
+
+	err = fs.MkdirAll("/foo/bar", 0755)
+	require.NoError(t, err)
+
+	info, err = fs.Stat("/foo/bar/dir")
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/")
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/foo")
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	info, err = fs.Stat("/foo/bar")
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	require.Equal(t, int64(0), info.Size())
+	require.Equal(t, true, info.IsDir())
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
+	require.NoError(t, err)
+
+	err = fs.MkdirAll("/foobar", 0755)
+	require.Error(t, err)
+}
+
+func testRename(t *testing.T, fs Filesystem) {
+	err := fs.Rename("/foobar", "/foobaz")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	err = fs.Rename("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+}
+
+func testRenameOverwrite(t *testing.T, fs Filesystem) {
+	_, err := fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+
+	err = fs.Rename("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+
+	data, err := fs.ReadFile("/foobaz")
+	require.NoError(t, err)
+	require.Equal(t, "foobar", string(data))
+}
+
+func testSymlink(t *testing.T, fs Filesystem) {
+	if _, ok := fs.(*s3Filesystem); ok {
+		return
+	}
+
+	err := fs.Symlink("/foobar", "/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	err = fs.Symlink("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	file := fs.Open("/foobaz")
+	require.NotNil(t, file)
+	require.Equal(t, "/foobaz", file.Name())
+
+	data := make([]byte, 10)
+	n, err := file.Read(data)
+	if err != nil {
+		if err != io.EOF {
+			require.NoError(t, err)
+		}
+	}
+	require.NoError(t, err)
+	require.Equal(t, 6, n)
+	require.Equal(t, "foobar", string(data[:n]))
+
+	stat, err := fs.Stat("/foobaz")
+	require.NoError(t, err)
+	require.Equal(t, "/foobaz", stat.Name())
+	require.Equal(t, int64(6), stat.Size())
+	require.NotEqual(t, 0, int(stat.Mode()&os.ModeSymlink))
+
+	link, ok := stat.IsLink()
+	require.Equal(t, "/foobar", link)
+	require.Equal(t, true, ok)
+
+	data, err = fs.ReadFile("/foobaz")
+	require.NoError(t, err)
+	require.Equal(t, "foobar", string(data))
+}
+
+func testSymlinkOpenStat(t *testing.T, fs Filesystem) {
+	if _, ok := fs.(*s3Filesystem); ok {
+		return
+	}
+
+	_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	err = fs.Symlink("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	file := fs.Open("/foobaz")
+	require.NotNil(t, file)
+	require.Equal(t, "/foobaz", file.Name())
+
+	fstat, err := file.Stat()
+	require.NoError(t, err)
+
+	stat, err := fs.Stat("/foobaz")
+	require.NoError(t, err)
+
+	require.Equal(t, "/foobaz", fstat.Name())
+	require.Equal(t, fstat.Name(), stat.Name())
+
+	require.Equal(t, int64(6), fstat.Size())
+	require.Equal(t, fstat.Size(), stat.Size())
+
+	require.NotEqual(t, 0, int(fstat.Mode()&os.ModeSymlink))
+	require.Equal(t, fstat.Mode(), stat.Mode())
+}
+
+func testStat(t *testing.T, fs Filesystem) {
+	_, _, err := fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	file := fs.Open("/foobar")
+	require.NotNil(t, file)
+
+	stat1, err := fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	stat2, err := file.Stat()
+	require.NoError(t, err)
+
+	require.Equal(t, stat1, stat2)
+}
+
+func testCopy(t *testing.T, fs Filesystem) {
+	err := fs.Rename("/foobar", "/foobaz")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("gduwotoxqb"))
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	err = fs.Copy("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+}
+
+func testCopyOverwrite(t *testing.T, fs Filesystem) {
+	_, err := fs.Stat("/foobar")
+	require.Error(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+
+	err = fs.Copy("/foobar", "/foobaz")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobar")
+	require.NoError(t, err)
+
+	_, err = fs.Stat("/foobaz")
+	require.NoError(t, err)
+
+	data, err := fs.ReadFile("/foobaz")
+	require.NoError(t, err)
+	require.Equal(t, "foobar", string(data))
+}
+
+func testSymlinkErrors(t *testing.T, fs Filesystem) {
+	if _, ok := fs.(*s3Filesystem); ok {
+		return
+	}
+
+	err := fs.Symlink("/foobar", "/foobaz")
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobar", strings.NewReader("foobar"))
+	require.NoError(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("foobaz"))
+	require.NoError(t, err)
+
+	err = fs.Symlink("/foobar", "/foobaz")
+	require.Error(t, err)
+
+	err = fs.Symlink("/foobar", "/bazfoo")
+	require.NoError(t, err)
+
+	err = fs.Symlink("/bazfoo", "/barfoo")
+	require.Error(t, err)
+}

io/fs/mem.go

@@ -4,7 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -15,28 +19,15 @@ import (
 // MemConfig is the config that is required for creating
 // a new memory filesystem.
 type MemConfig struct {
-	// Namee is the name of the filesystem
-	Name string
-
-	// Base is the base path to be reported for this filesystem
-	Base string
-
-	// Size is the capacity of the filesystem in bytes
-	Size int64
-
-	// Set true to automatically delete the oldest files until there's
-	// enough space to store a new file
-	Purge bool
-
-	// For logging, optional
-	Logger log.Logger
+	Logger log.Logger // For logging, optional
 }
 
 type memFileInfo struct {
-	name    string
-	size    int64
-	lastMod time.Time
-	linkTo  string
+	name    string    // Full name of the file (including path)
+	size    int64     // The size of the file in bytes
+	dir     bool      // Whether this file represents a directory
+	lastMod time.Time // The time of the last modification of the file
+	linkTo  string    // Where the file links to, empty if it's not a link
 }
 
 func (f *memFileInfo) Name() string {
@@ -47,6 +38,20 @@ func (f *memFileInfo) Size() int64 {
 	return f.size
 }
 
+func (f *memFileInfo) Mode() fs.FileMode {
+	mode := fs.FileMode(fs.ModePerm)
+
+	if f.dir {
+		mode |= fs.ModeDir
+	}
+
+	if len(f.linkTo) != 0 {
+		mode |= fs.ModeSymlink
+	}
+
+	return mode
+}
+
 func (f *memFileInfo) ModTime() time.Time {
 	return f.lastMod
 }
@@ -56,24 +61,12 @@ func (f *memFileInfo) IsLink() (string, bool) {
 }
 
 func (f *memFileInfo) IsDir() bool {
-	return false
+	return f.dir
 }
 
 type memFile struct {
-	// Name of the file
-	name string
-
-	// Size of the file in bytes
-	size int64
-
-	// Last modification of the file as a UNIX timestamp
-	lastMod time.Time
-
-	// Contents of the file
-	data *bytes.Buffer
-
-	// Link to another file
-	linkTo string
+	memFileInfo
+	data *bytes.Buffer // Contents of the file
 }
 
 func (f *memFile) Name() string {
@@ -84,6 +77,7 @@ func (f *memFile) Stat() (FileInfo, error) {
 	info := &memFileInfo{
 		name:    f.name,
 		size:    f.size,
+		dir:     f.dir,
 		lastMod: f.lastMod,
 		linkTo:  f.linkTo,
 	}
@@ -110,8 +104,8 @@ func (f *memFile) Close() error {
 }
 
 type memFilesystem struct {
-	name string
-	base string
+	metadata map[string]string
+	metaLock sync.RWMutex
 
 	// Mapping of path to file
 	files map[string]*memFile
@@ -122,29 +116,19 @@ type memFilesystem struct {
 	// Pool for the storage of the contents of files
 	dataPool sync.Pool
 
-	// Max. size of the filesystem in bytes as
-	// given by the config
-	maxSize int64
-
 	// Current size of the filesystem in bytes
 	currentSize int64
 
-	// Purge setting from the config
-	purge bool
-
 	// Logger from the config
 	logger log.Logger
 }
 
 // NewMemFilesystem creates a new filesystem in memory that implements
 // the Filesystem interface.
-func NewMemFilesystem(config MemConfig) Filesystem {
+func NewMemFilesystem(config MemConfig) (Filesystem, error) {
 	fs := &memFilesystem{
-		name:    config.Name,
-		base:    config.Base,
-		maxSize: config.Size,
-		purge:   config.Purge,
-		logger:  config.Logger,
+		metadata: make(map[string]string),
+		logger:   config.Logger,
 	}
 
 	if fs.logger == nil {
@@ -161,70 +145,105 @@ func NewMemFilesystem(config MemConfig) Filesystem {
 		},
 	}
 
-	fs.logger.WithFields(log.Fields{
-		"name":       fs.name,
-		"size_bytes": fs.maxSize,
-		"purge":      fs.purge,
-	}).Debug().Log("Created")
+	fs.logger.Debug().Log("Created")
 
-	return fs
+	return fs, nil
+}
+
+func NewMemFilesystemFromDir(dir string, config MemConfig) (Filesystem, error) {
+	mem, err := NewMemFilesystem(config)
+	if err != nil {
+		return nil, err
+	}
+
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		mode := info.Mode()
+		if !mode.IsRegular() {
+			return nil
+		}
+
+		if mode&os.ModeSymlink != 0 {
+			return nil
+		}
+
+		file, err := os.Open(path)
+		if err != nil {
+			return nil
+		}
+
+		defer file.Close()
+
+		_, _, err = mem.WriteFileReader(path, file)
+		if err != nil {
+			return fmt.Errorf("can't copy %s", path)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return mem, nil
 }
 
 func (fs *memFilesystem) Name() string {
-	return fs.name
-}
-
-func (fs *memFilesystem) Base() string {
-	return fs.base
-}
-
-func (fs *memFilesystem) Rebase(base string) error {
-	fs.base = base
-
-	return nil
+	return "mem"
 }
 
 func (fs *memFilesystem) Type() string {
-	return "memfs"
+	return "mem"
+}
+
+func (fs *memFilesystem) Metadata(key string) string {
+	fs.metaLock.RLock()
+	defer fs.metaLock.RUnlock()
+
+	return fs.metadata[key]
+}
+
+func (fs *memFilesystem) SetMetadata(key, data string) {
+	fs.metaLock.Lock()
+	defer fs.metaLock.Unlock()
+
+	fs.metadata[key] = data
 }
 
 func (fs *memFilesystem) Size() (int64, int64) {
 	fs.filesLock.RLock()
 	defer fs.filesLock.RUnlock()
 
-	return fs.currentSize, fs.maxSize
-}
-
-func (fs *memFilesystem) Resize(size int64) {
-	fs.filesLock.Lock()
-	defer fs.filesLock.Unlock()
-
-	diffSize := fs.maxSize - size
-
-	if diffSize == 0 {
-		return
-	}
-
-	if diffSize > 0 {
-		fs.free(diffSize)
-	}
-
-	fs.logger.WithFields(log.Fields{
-		"from_bytes": fs.maxSize,
-		"to_bytes":   size,
-	}).Debug().Log("Resizing")
-
-	fs.maxSize = size
+	return fs.currentSize, -1
 }
 
 func (fs *memFilesystem) Files() int64 {
 	fs.filesLock.RLock()
 	defer fs.filesLock.RUnlock()
 
-	return int64(len(fs.files))
+	nfiles := int64(0)
+
+	for _, f := range fs.files {
+		if f.dir {
+			continue
+		}
+
+		nfiles++
+	}
+
+	return nfiles
 }
 
 func (fs *memFilesystem) Open(path string) File {
+	path = fs.cleanPath(path)
+
 	fs.filesLock.RLock()
 	file, ok := fs.files[path]
 	fs.filesLock.RUnlock()
@@ -234,29 +253,68 @@ func (fs *memFilesystem) Open(path string) File {
 	}
 
 	newFile := &memFile{
-		name:    file.name,
-		size:    file.size,
-		lastMod: file.lastMod,
-		linkTo:  file.linkTo,
+		memFileInfo: memFileInfo{
+			name:    file.name,
+			size:    file.size,
+			lastMod: file.lastMod,
+			linkTo:  file.linkTo,
+		},
+	}
+
+	if len(file.linkTo) != 0 {
+		file, ok = fs.files[file.linkTo]
+		if !ok {
+			return nil
+		}
 	}
 
 	if file.data != nil {
+		newFile.lastMod = file.lastMod
 		newFile.data = bytes.NewBuffer(file.data.Bytes())
+		newFile.size = int64(newFile.data.Len())
 	}
 
 	return newFile
 }
 
+func (fs *memFilesystem) ReadFile(path string) ([]byte, error) {
+	path = fs.cleanPath(path)
+
+	fs.filesLock.RLock()
+	file, ok := fs.files[path]
+	fs.filesLock.RUnlock()
+
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+
+	if len(file.linkTo) != 0 {
+		file, ok = fs.files[file.linkTo]
+		if !ok {
+			return nil, os.ErrNotExist
+		}
+	}
+
+	if file.data != nil {
+		return file.data.Bytes(), nil
+	}
+
+	return nil, nil
+}
+
 func (fs *memFilesystem) Symlink(oldname, newname string) error {
+	oldname = fs.cleanPath(oldname)
+	newname = fs.cleanPath(newname)
+
 	fs.filesLock.Lock()
 	defer fs.filesLock.Unlock()
 
-	if _, ok := fs.files[newname]; ok {
-		return fmt.Errorf("%s already exist", newname)
+	if _, ok := fs.files[oldname]; !ok {
+		return os.ErrNotExist
 	}
 
-	if oldname[0] != '/' {
-		oldname = "/" + oldname
+	if _, ok := fs.files[newname]; ok {
+		return os.ErrExist
 	}
 
 	if file, ok := fs.files[oldname]; ok {
@@ -266,11 +324,14 @@ func (fs *memFilesystem) Symlink(oldname, newname string) error {
 	}
 
 	newFile := &memFile{
-		name:    newname,
-		size:    0,
-		lastMod: time.Now(),
-		data:    nil,
-		linkTo:  oldname,
+		memFileInfo: memFileInfo{
+			name:    newname,
+			dir:     false,
+			size:    0,
+			lastMod: time.Now(),
+			linkTo:  oldname,
+		},
+		data: nil,
 	}
 
 	fs.files[newname] = newFile
@@ -278,18 +339,21 @@ func (fs *memFilesystem) Symlink(oldname, newname string) error {
 	return nil
 }
 
-func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
+func (fs *memFilesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
+	path = fs.cleanPath(path)
+
 	newFile := &memFile{
-		name:    path,
-		size:    0,
-		lastMod: time.Now(),
-		data:    nil,
+		memFileInfo: memFileInfo{
+			name:    path,
+			dir:     false,
+			size:    0,
+			lastMod: time.Now(),
+		},
+		data: fs.dataPool.Get().(*bytes.Buffer),
 	}
 
-	data := fs.dataPool.Get().(*bytes.Buffer)
-	data.Reset()
-
-	size, err := data.ReadFrom(r)
+	newFile.data.Reset()
+	size, err := newFile.data.ReadFrom(r)
 	if err != nil {
 		fs.logger.WithFields(log.Fields{
 			"path":           path,
@@ -297,55 +361,26 @@ func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
 			"error":          err,
 		}).Warn().Log("Incomplete file")
 	}
-	newFile.size = size
-	newFile.data = data
 
-	// reject if the new file is larger than the available space
-	if fs.maxSize > 0 && newFile.size > fs.maxSize {
-		fs.dataPool.Put(data)
-		return -1, false, fmt.Errorf("File is too big")
-	}
+	newFile.size = size
 
 	fs.filesLock.Lock()
 	defer fs.filesLock.Unlock()
 
-	// calculate the new size of the filesystem
-	newSize := fs.currentSize + newFile.size
-
 	file, replace := fs.files[path]
 	if replace {
-		newSize -= file.size
+		delete(fs.files, path)
+
+		fs.currentSize -= file.size
+
+		fs.dataPool.Put(file.data)
+		file.data = nil
 	}
 
-	if fs.maxSize > 0 {
-		if newSize > fs.maxSize {
-			if !fs.purge {
-				fs.dataPool.Put(data)
-				return -1, false, fmt.Errorf("not enough space on device")
-			}
-
-			if replace {
-				delete(fs.files, path)
-				fs.currentSize -= file.size
-
-				fs.dataPool.Put(file.data)
-				file.data = nil
-			}
-
-			newSize -= fs.free(fs.currentSize + newFile.size - fs.maxSize)
-		}
-	} else {
-		if replace {
-			delete(fs.files, path)
-
-			fs.dataPool.Put(file.data)
-			file.data = nil
-		}
-	}
-
-	fs.currentSize = newSize
 	fs.files[path] = newFile
 
+	fs.currentSize += newFile.size
+
 	logger := fs.logger.WithFields(log.Fields{
 		"path":           newFile.name,
 		"filesize_bytes": newFile.size,
@@ -361,7 +396,18 @@ func (fs *memFilesystem) Store(path string, r io.Reader) (int64, bool, error) {
 	return newFile.size, !replace, nil
 }
 
-func (fs *memFilesystem) free(size int64) int64 {
+func (fs *memFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
+	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (fs *memFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
+	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (fs *memFilesystem) Purge(size int64) int64 {
+	fs.filesLock.Lock()
+	defer fs.filesLock.Unlock()
+
 	files := []*memFile{}
 
 	for _, f := range fs.files {
@@ -399,7 +445,190 @@ func (fs *memFilesystem) free(size int64) int64 {
 	return freed
 }
 
-func (fs *memFilesystem) Delete(path string) int64 {
+func (fs *memFilesystem) MkdirAll(path string, perm os.FileMode) error {
+	path = fs.cleanPath(path)
+
+	fs.filesLock.Lock()
+	defer fs.filesLock.Unlock()
+
+	info, err := fs.stat(path)
+	if err == nil {
+		if info.IsDir() {
+			return nil
+		}
+
+		return os.ErrExist
+	}
+
+	f := &memFile{
+		memFileInfo: memFileInfo{
+			name:    path,
+			size:    0,
+			dir:     true,
+			lastMod: time.Now(),
+		},
+		data: nil,
+	}
+
+	fs.files[path] = f
+
+	return nil
+}
+
+func (fs *memFilesystem) Rename(src, dst string) error {
+	src = filepath.Join("/", filepath.Clean(src))
+	dst = filepath.Join("/", filepath.Clean(dst))
+
+	if src == dst {
+		return nil
+	}
+
+	fs.filesLock.Lock()
+	defer fs.filesLock.Unlock()
+
+	srcFile, ok := fs.files[src]
+	if !ok {
+		return os.ErrNotExist
+	}
+
+	dstFile, ok := fs.files[dst]
+	if ok {
+		fs.currentSize -= dstFile.size
+
+		fs.dataPool.Put(dstFile.data)
+		dstFile.data = nil
+	}
+
+	fs.files[dst] = srcFile
+	delete(fs.files, src)
+
+	return nil
+}
+
+func (fs *memFilesystem) Copy(src, dst string) error {
+	src = filepath.Join("/", filepath.Clean(src))
+	dst = filepath.Join("/", filepath.Clean(dst))
+
+	if src == dst {
+		return nil
+	}
+
+	fs.filesLock.Lock()
+	defer fs.filesLock.Unlock()
+
+	srcFile, ok := fs.files[src]
+	if !ok {
+		return os.ErrNotExist
+	}
+
+	if srcFile.dir {
+		return os.ErrNotExist
+	}
+
+	if fs.isDir(dst) {
+		return os.ErrInvalid
+	}
+
+	dstFile, ok := fs.files[dst]
+	if ok {
+		fs.currentSize -= dstFile.size
+	} else {
+		dstFile = &memFile{
+			memFileInfo: memFileInfo{
+				name:    dst,
+				dir:     false,
+				size:    srcFile.size,
+				lastMod: time.Now(),
+			},
+			data: fs.dataPool.Get().(*bytes.Buffer),
+		}
+	}
+
+	dstFile.data.Reset()
+	dstFile.data.Write(srcFile.data.Bytes())
+
+	fs.currentSize += dstFile.size
+
+	fs.files[dst] = dstFile
+
+	return nil
+}
+
+func (fs *memFilesystem) Stat(path string) (FileInfo, error) {
+	path = fs.cleanPath(path)
+
+	fs.filesLock.RLock()
+	defer fs.filesLock.RUnlock()
+
+	return fs.stat(path)
+}
+
+func (fs *memFilesystem) stat(path string) (FileInfo, error) {
+	file, ok := fs.files[path]
+	if ok {
+		f := &memFileInfo{
+			name:    file.name,
+			size:    file.size,
+			dir:     file.dir,
+			lastMod: file.lastMod,
+			linkTo:  file.linkTo,
+		}
+
+		if len(f.linkTo) != 0 {
+			file, ok := fs.files[f.linkTo]
+			if !ok {
+				return nil, os.ErrNotExist
+			}
+
+			f.lastMod = file.lastMod
+			f.size = file.size
+		}
+
+		return f, nil
+	}
+
+	// Check for directories
+	if !fs.isDir(path) {
+		return nil, os.ErrNotExist
+	}
+
+	f := &memFileInfo{
+		name:    path,
+		size:    0,
+		dir:     true,
+		lastMod: time.Now(),
+		linkTo:  "",
+	}
+
+	return f, nil
+}
+
+func (fs *memFilesystem) isDir(path string) bool {
+	file, ok := fs.files[path]
+	if ok {
+		return file.dir
+	}
+
+	if !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+
+	if path == "/" {
+		return true
+	}
+
+	for k := range fs.files {
+		if strings.HasPrefix(k, path) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (fs *memFilesystem) Remove(path string) int64 {
+	path = fs.cleanPath(path)
+
 	fs.filesLock.Lock()
 	defer fs.filesLock.Unlock()
 
@@ -423,7 +652,7 @@ func (fs *memFilesystem) Delete(path string) int64 {
 	return file.size
 }
 
-func (fs *memFilesystem) DeleteAll() int64 {
+func (fs *memFilesystem) RemoveAll() int64 {
 	fs.filesLock.Lock()
 	defer fs.filesLock.Unlock()
 
@@ -435,19 +664,28 @@ func (fs *memFilesystem) DeleteAll() int64 {
 	return size
 }
 
-func (fs *memFilesystem) List(pattern string) []FileInfo {
+func (fs *memFilesystem) List(path, pattern string) []FileInfo {
+	path = fs.cleanPath(path)
 	files := []FileInfo{}
 
 	fs.filesLock.RLock()
 	defer fs.filesLock.RUnlock()
 
 	for _, file := range fs.files {
+		if !strings.HasPrefix(file.name, path) {
+			continue
+		}
+
 		if len(pattern) != 0 {
 			if ok, _ := glob.Match(pattern, file.name, '/'); !ok {
 				continue
 			}
 		}
 
+		if file.dir {
+			continue
+		}
+
 		files = append(files, &memFileInfo{
 			name:    file.name,
 			size:    file.size,
@@ -458,3 +696,11 @@ func (fs *memFilesystem) List(pattern string) []FileInfo {
 
 	return files
 }
+
+func (fs *memFilesystem) cleanPath(path string) string {
+	if !filepath.IsAbs(path) {
+		path = filepath.Join("/", path)
+	}
+
+	return filepath.Join("/", filepath.Clean(path))
+}

io/fs/mem_test.go

@@ -1,406 +1,30 @@
 package fs
 
 import (
-	"strings"
 	"testing"
-	"time"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func TestNew(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
+func TestMemFromDir(t *testing.T) {
+	mem, err := NewMemFilesystemFromDir(".", MemConfig{})
+	require.NoError(t, err)
 
-	cur, max := mem.Size()
+	names := []string{}
+	for _, f := range mem.List("/", "/*.go") {
+		names = append(names, f.Name())
+	}
 
-	assert.Equal(t, int64(0), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(0), cur)
-}
-
-func TestSimplePutNoPurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data := strings.NewReader("xxxxx")
-
-	size, created, err := mem.Store("/foobar", data)
-
-	assert.Nil(t, err)
-	assert.Equal(t, int64(5), size)
-	assert.Equal(t, true, created)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(5), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(1), cur)
-}
-
-func TestSimpleDelete(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	size := mem.Delete("/foobar")
-
-	assert.Equal(t, int64(-1), size)
-
-	data := strings.NewReader("xxxxx")
-
-	mem.Store("/foobar", data)
-
-	size = mem.Delete("/foobar")
-
-	assert.Equal(t, int64(5), size)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(0), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(0), cur)
-}
-
-func TestReplaceNoPurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data := strings.NewReader("xxxxx")
-
-	size, created, err := mem.Store("/foobar", data)
-
-	assert.Nil(t, err)
-	assert.Equal(t, int64(5), size)
-	assert.Equal(t, true, created)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(5), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(1), cur)
-
-	data = strings.NewReader("yyy")
-
-	size, created, err = mem.Store("/foobar", data)
-
-	assert.Nil(t, err)
-	assert.Equal(t, int64(3), size)
-	assert.Equal(t, false, created)
-
-	cur, max = mem.Size()
-
-	assert.Equal(t, int64(3), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(1), cur)
-}
-
-func TestReplacePurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: true,
-	})
-
-	data1 := strings.NewReader("xxx")
-	data2 := strings.NewReader("yyy")
-	data3 := strings.NewReader("zzz")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-	mem.Store("/foobar3", data3)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(9), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(3), cur)
-
-	data4 := strings.NewReader("zzzzz")
-
-	size, _, _ := mem.Store("/foobar1", data4)
-
-	assert.Equal(t, int64(5), size)
-
-	cur, max = mem.Size()
-
-	assert.Equal(t, int64(8), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(2), cur)
-}
-
-func TestReplaceUnlimited(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  0,
-		Purge: false,
-	})
-
-	data := strings.NewReader("xxxxx")
-
-	size, created, err := mem.Store("/foobar", data)
-
-	assert.Nil(t, err)
-	assert.Equal(t, int64(5), size)
-	assert.Equal(t, true, created)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(5), cur)
-	assert.Equal(t, int64(0), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(1), cur)
-
-	data = strings.NewReader("yyy")
-
-	size, created, err = mem.Store("/foobar", data)
-
-	assert.Nil(t, err)
-	assert.Equal(t, int64(3), size)
-	assert.Equal(t, false, created)
-
-	cur, max = mem.Size()
-
-	assert.Equal(t, int64(3), cur)
-	assert.Equal(t, int64(0), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(1), cur)
-}
-
-func TestTooBigNoPurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data := strings.NewReader("xxxxxyyyyyz")
-
-	size, _, _ := mem.Store("/foobar", data)
-
-	assert.Equal(t, int64(-1), size)
-}
-
-func TestTooBigPurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: true,
-	})
-
-	data1 := strings.NewReader("xxxxx")
-	data2 := strings.NewReader("yyyyy")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-
-	data := strings.NewReader("xxxxxyyyyyz")
-
-	size, _, _ := mem.Store("/foobar", data)
-
-	assert.Equal(t, int64(-1), size)
-}
-
-func TestFullSpaceNoPurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data1 := strings.NewReader("xxxxx")
-	data2 := strings.NewReader("yyyyy")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(10), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(2), cur)
-
-	data3 := strings.NewReader("zzzzz")
-
-	size, _, _ := mem.Store("/foobar3", data3)
-
-	assert.Equal(t, int64(-1), size)
-}
-
-func TestFullSpacePurge(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: true,
-	})
-
-	data1 := strings.NewReader("xxxxx")
-	data2 := strings.NewReader("yyyyy")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(10), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(2), cur)
-
-	data3 := strings.NewReader("zzzzz")
-
-	size, _, _ := mem.Store("/foobar3", data3)
-
-	assert.Equal(t, int64(5), size)
-
-	cur, max = mem.Size()
-
-	assert.Equal(t, int64(10), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(2), cur)
-}
-
-func TestFullSpacePurgeMulti(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: true,
-	})
-
-	data1 := strings.NewReader("xxx")
-	data2 := strings.NewReader("yyy")
-	data3 := strings.NewReader("zzz")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-	mem.Store("/foobar3", data3)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(9), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(3), cur)
-
-	data4 := strings.NewReader("zzzzz")
-
-	size, _, _ := mem.Store("/foobar4", data4)
-
-	assert.Equal(t, int64(5), size)
-
-	cur, max = mem.Size()
-
-	assert.Equal(t, int64(8), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(2), cur)
-}
-
-func TestPurgeOrder(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: true,
-	})
-
-	data1 := strings.NewReader("xxxxx")
-	data2 := strings.NewReader("yyyyy")
-	data3 := strings.NewReader("zzzzz")
-
-	mem.Store("/foobar1", data1)
-	time.Sleep(1 * time.Second)
-	mem.Store("/foobar2", data2)
-	time.Sleep(1 * time.Second)
-	mem.Store("/foobar3", data3)
-
-	file := mem.Open("/foobar1")
-
-	assert.Nil(t, file)
-}
-
-func TestList(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data1 := strings.NewReader("a")
-	data2 := strings.NewReader("bb")
-	data3 := strings.NewReader("ccc")
-	data4 := strings.NewReader("dddd")
-
-	mem.Store("/foobar1", data1)
-	mem.Store("/foobar2", data2)
-	mem.Store("/foobar3", data3)
-	mem.Store("/foobar4", data4)
-
-	cur, max := mem.Size()
-
-	assert.Equal(t, int64(10), cur)
-	assert.Equal(t, int64(10), max)
-
-	cur = mem.Files()
-
-	assert.Equal(t, int64(4), cur)
-
-	files := mem.List("")
-
-	assert.Equal(t, 4, len(files))
-}
-
-func TestData(t *testing.T) {
-	mem := NewMemFilesystem(MemConfig{
-		Size:  10,
-		Purge: false,
-	})
-
-	data := "gduwotoxqb"
-
-	data1 := strings.NewReader(data)
-
-	mem.Store("/foobar", data1)
-
-	file := mem.Open("/foobar")
-
-	data2 := make([]byte, len(data)+1)
-	n, _ := file.Read(data2)
-
-	assert.Equal(t, len(data), n)
-	assert.Equal(t, []byte(data), data2[:n])
+	require.ElementsMatch(t, []string{
+		"/disk.go",
+		"/fs_test.go",
+		"/fs.go",
+		"/mem_test.go",
+		"/mem.go",
+		"/readonly_test.go",
+		"/readonly.go",
+		"/s3.go",
+		"/sized_test.go",
+		"/sized.go",
+	}, names)
 }

io/fs/readonly.go

@@ -0,0 +1,54 @@
+package fs
+
+import (
+	"io"
+	"os"
+)
+
+type readOnlyFilesystem struct {
+	Filesystem
+}
+
+func NewReadOnlyFilesystem(fs Filesystem) (Filesystem, error) {
+	r := &readOnlyFilesystem{
+		Filesystem: fs,
+	}
+
+	return r, nil
+}
+
+func (r *readOnlyFilesystem) Symlink(oldname, newname string) error {
+	return os.ErrPermission
+}
+
+func (r *readOnlyFilesystem) WriteFileReader(path string, rd io.Reader) (int64, bool, error) {
+	return -1, false, os.ErrPermission
+}
+
+func (r *readOnlyFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
+	return -1, false, os.ErrPermission
+}
+
+func (r *readOnlyFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
+	return -1, false, os.ErrPermission
+}
+
+func (r *readOnlyFilesystem) MkdirAll(path string, perm os.FileMode) error {
+	return os.ErrPermission
+}
+
+func (r *readOnlyFilesystem) Remove(path string) int64 {
+	return -1
+}
+
+func (r *readOnlyFilesystem) RemoveAll() int64 {
+	return 0
+}
+
+func (r *readOnlyFilesystem) Purge(size int64) int64 {
+	return 0
+}
+
+func (r *readOnlyFilesystem) Resize(size int64) error {
+	return os.ErrPermission
+}

io/fs/readonly_test.go

@@ -0,0 +1,50 @@
+package fs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadOnly(t *testing.T) {
+	mem, err := NewMemFilesystemFromDir(".", MemConfig{})
+	require.NoError(t, err)
+
+	ro, err := NewReadOnlyFilesystem(mem)
+	require.NoError(t, err)
+
+	err = ro.Symlink("/readonly.go", "/foobar.go")
+	require.Error(t, err)
+
+	_, _, err = ro.WriteFile("/readonly.go", []byte("foobar"))
+	require.Error(t, err)
+
+	_, _, err = ro.WriteFileReader("/readonly.go", strings.NewReader("foobar"))
+	require.Error(t, err)
+
+	_, _, err = ro.WriteFileSafe("/readonly.go", []byte("foobar"))
+	require.Error(t, err)
+
+	err = ro.MkdirAll("/foobar/baz", 0700)
+	require.Error(t, err)
+
+	res := ro.Remove("/readonly.go")
+	require.Equal(t, int64(-1), res)
+
+	res = ro.RemoveAll()
+	require.Equal(t, int64(0), res)
+
+	rop, ok := ro.(PurgeFilesystem)
+	require.True(t, ok, "must implement PurgeFilesystem")
+
+	size, _ := ro.Size()
+	res = rop.Purge(size)
+	require.Equal(t, int64(0), res)
+
+	ros, ok := ro.(SizedFilesystem)
+	require.True(t, ok, "must implement SizedFilesystem")
+
+	err = ros.Resize(100)
+	require.Error(t, err)
+}

io/fs/s3.go

@@ -1,9 +1,15 @@
 package fs
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/datarhei/core/v16/glob"
@@ -15,7 +21,6 @@ import (
 type S3Config struct {
 	// Namee is the name of the filesystem
 	Name            string
-	Base            string
 	Endpoint        string
 	AccessKeyID     string
 	SecretAccessKey string
@@ -26,9 +31,11 @@ type S3Config struct {
 	Logger log.Logger
 }
 
-type s3fs struct {
+type s3Filesystem struct {
+	metadata map[string]string
+	metaLock sync.RWMutex
+
 	name string
-	base string
 
 	endpoint        string
 	accessKeyID     string
@@ -42,10 +49,12 @@ type s3fs struct {
 	logger log.Logger
 }
 
+var fakeDirEntry = "..."
+
 func NewS3Filesystem(config S3Config) (Filesystem, error) {
-	fs := &s3fs{
+	fs := &s3Filesystem{
+		metadata:        make(map[string]string),
 		name:            config.Name,
-		base:            config.Base,
 		endpoint:        config.Endpoint,
 		accessKeyID:     config.AccessKeyID,
 		secretAccessKey: config.SecretAccessKey,
@@ -106,28 +115,32 @@ func NewS3Filesystem(config S3Config) (Filesystem, error) {
 	return fs, nil
 }
 
-func (fs *s3fs) Name() string {
+func (fs *s3Filesystem) Name() string {
 	return fs.name
 }
 
-func (fs *s3fs) Base() string {
-	return fs.base
+func (fs *s3Filesystem) Type() string {
+	return "s3"
 }
 
-func (fs *s3fs) Rebase(base string) error {
-	fs.base = base
+func (fs *s3Filesystem) Metadata(key string) string {
+	fs.metaLock.RLock()
+	defer fs.metaLock.RUnlock()
 
-	return nil
+	return fs.metadata[key]
 }
 
-func (fs *s3fs) Type() string {
-	return "s3fs"
+func (fs *s3Filesystem) SetMetadata(key, data string) {
+	fs.metaLock.Lock()
+	defer fs.metaLock.Unlock()
+
+	fs.metadata[key] = data
 }
 
-func (fs *s3fs) Size() (int64, int64) {
+func (fs *s3Filesystem) Size() (int64, int64) {
 	size := int64(0)
 
-	files := fs.List("")
+	files := fs.List("/", "")
 
 	for _, file := range files {
 		size += file.Size()
@@ -136,14 +149,18 @@ func (fs *s3fs) Size() (int64, int64) {
 	return size, -1
 }
 
-func (fs *s3fs) Resize(size int64) {}
-
-func (fs *s3fs) Files() int64 {
+func (fs *s3Filesystem) Files() int64 {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
-		Recursive: true,
+		WithVersions: false,
+		WithMetadata: false,
+		Prefix:       "",
+		Recursive:    true,
+		MaxKeys:      0,
+		StartAfter:   "",
+		UseV1:        false,
 	})
 
 	nfiles := int64(0)
@@ -152,19 +169,77 @@ func (fs *s3fs) Files() int64 {
 		if object.Err != nil {
 			fs.logger.WithError(object.Err).Log("Listing object failed")
 		}
+
+		if strings.HasSuffix("/"+object.Key, "/"+fakeDirEntry) {
+			// Skip fake entries (see MkdirAll)
+			continue
+		}
+
 		nfiles++
 	}
 
 	return nfiles
 }
 
-func (fs *s3fs) Symlink(oldname, newname string) error {
+func (fs *s3Filesystem) Symlink(oldname, newname string) error {
 	return fmt.Errorf("not implemented")
 }
 
-func (fs *s3fs) Open(path string) File {
-	//ctx, cancel := context.WithCancel(context.Background())
-	//defer cancel()
+func (fs *s3Filesystem) Stat(path string) (FileInfo, error) {
+	path = fs.cleanPath(path)
+
+	if len(path) == 0 {
+		return &s3FileInfo{
+			name:         "/",
+			size:         0,
+			dir:          true,
+			lastModified: time.Now(),
+		}, nil
+	}
+
+	ctx := context.Background()
+
+	object, err := fs.client.GetObject(ctx, fs.bucket, path, minio.GetObjectOptions{})
+	if err != nil {
+		if fs.isDir(path) {
+			return &s3FileInfo{
+				name:         "/" + path,
+				size:         0,
+				dir:          true,
+				lastModified: time.Now(),
+			}, nil
+		}
+
+		fs.logger.Debug().WithField("key", path).WithError(err).Log("Not found")
+		return nil, err
+	}
+
+	defer object.Close()
+
+	stat, err := object.Stat()
+	if err != nil {
+		if fs.isDir(path) {
+			return &s3FileInfo{
+				name:         "/" + path,
+				size:         0,
+				dir:          true,
+				lastModified: time.Now(),
+			}, nil
+		}
+
+		fs.logger.Debug().WithField("key", path).WithError(err).Log("Stat failed")
+		return nil, err
+	}
+
+	return &s3FileInfo{
+		name:         "/" + stat.Key,
+		size:         stat.Size,
+		lastModified: stat.LastModified,
+	}, nil
+}
+
+func (fs *s3Filesystem) Open(path string) File {
+	path = fs.cleanPath(path)
 	ctx := context.Background()
 
 	object, err := fs.client.GetObject(ctx, fs.bucket, path, minio.GetObjectOptions{})
@@ -181,7 +256,7 @@ func (fs *s3fs) Open(path string) File {
 
 	file := &s3File{
 		data:         object,
-		name:         stat.Key,
+		name:         "/" + stat.Key,
 		size:         stat.Size,
 		lastModified: stat.LastModified,
 	}
@@ -191,7 +266,26 @@ func (fs *s3fs) Open(path string) File {
 	return file
 }
 
-func (fs *s3fs) Store(path string, r io.Reader) (int64, bool, error) {
+func (fs *s3Filesystem) ReadFile(path string) ([]byte, error) {
+	path = fs.cleanPath(path)
+	file := fs.Open(path)
+	if file == nil {
+		return nil, os.ErrNotExist
+	}
+
+	defer file.Close()
+
+	buf := &bytes.Buffer{}
+
+	_, err := buf.ReadFrom(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func (fs *s3Filesystem) write(path string, r io.Reader) (int64, bool, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -234,10 +328,84 @@ func (fs *s3fs) Store(path string, r io.Reader) (int64, bool, error) {
 		"overwrite": overwrite,
 	}).Log("Stored")
 
-	return info.Size, overwrite, nil
+	return info.Size, !overwrite, nil
 }
 
-func (fs *s3fs) Delete(path string) int64 {
+func (fs *s3Filesystem) WriteFileReader(path string, r io.Reader) (int64, bool, error) {
+	path = fs.cleanPath(path)
+	return fs.write(path, r)
+}
+
+func (fs *s3Filesystem) WriteFile(path string, data []byte) (int64, bool, error) {
+	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (fs *s3Filesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
+	return fs.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (fs *s3Filesystem) Rename(src, dst string) error {
+	src = fs.cleanPath(src)
+	dst = fs.cleanPath(dst)
+
+	err := fs.Copy(src, dst)
+	if err != nil {
+		return err
+	}
+
+	res := fs.Remove(src)
+	if res == -1 {
+		return fmt.Errorf("failed to remove source file: %s", src)
+	}
+
+	return nil
+}
+
+func (fs *s3Filesystem) Copy(src, dst string) error {
+	src = fs.cleanPath(src)
+	dst = fs.cleanPath(dst)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	_, err := fs.client.CopyObject(ctx, minio.CopyDestOptions{
+		Bucket: fs.bucket,
+		Object: dst,
+	}, minio.CopySrcOptions{
+		Bucket: fs.bucket,
+		Object: src,
+	})
+
+	return err
+}
+
+func (fs *s3Filesystem) MkdirAll(path string, perm os.FileMode) error {
+	if path == "/" {
+		return nil
+	}
+
+	info, err := fs.Stat(path)
+	if err == nil {
+		if !info.IsDir() {
+			return os.ErrExist
+		}
+
+		return nil
+	}
+
+	path = filepath.Join(path, fakeDirEntry)
+
+	_, _, err = fs.write(path, strings.NewReader(""))
+	if err != nil {
+		return fmt.Errorf("can't create directory")
+	}
+
+	return nil
+}
+
+func (fs *s3Filesystem) Remove(path string) int64 {
+	path = fs.cleanPath(path)
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -260,7 +428,7 @@ func (fs *s3fs) Delete(path string) int64 {
 	return stat.Size
 }
 
-func (fs *s3fs) DeleteAll() int64 {
+func (fs *s3Filesystem) RemoveAll() int64 {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -295,14 +463,16 @@ func (fs *s3fs) DeleteAll() int64 {
 	return totalSize
 }
 
-func (fs *s3fs) List(pattern string) []FileInfo {
+func (fs *s3Filesystem) List(path, pattern string) []FileInfo {
+	path = fs.cleanPath(path)
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
 		WithVersions: false,
 		WithMetadata: false,
-		Prefix:       "",
+		Prefix:       path,
 		Recursive:    true,
 		MaxKeys:      0,
 		StartAfter:   "",
@@ -317,14 +487,20 @@ func (fs *s3fs) List(pattern string) []FileInfo {
 			continue
 		}
 
+		key := "/" + object.Key
+		if strings.HasSuffix(key, "/"+fakeDirEntry) {
+			// filter out fake directory entries (see MkdirAll)
+			continue
+		}
+
 		if len(pattern) != 0 {
-			if ok, _ := glob.Match(pattern, object.Key, '/'); !ok {
+			if ok, _ := glob.Match(pattern, key, '/'); !ok {
 				continue
 			}
 		}
 
 		f := &s3FileInfo{
-			name:         object.Key,
+			name:         key,
 			size:         object.Size,
 			lastModified: object.LastModified,
 		}
@@ -335,9 +511,56 @@ func (fs *s3fs) List(pattern string) []FileInfo {
 	return files
 }
 
+func (fs *s3Filesystem) isDir(path string) bool {
+	if !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+
+	if path == "/" {
+		return true
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	ch := fs.client.ListObjects(ctx, fs.bucket, minio.ListObjectsOptions{
+		WithVersions: false,
+		WithMetadata: false,
+		Prefix:       path,
+		Recursive:    true,
+		MaxKeys:      1,
+		StartAfter:   "",
+		UseV1:        false,
+	})
+
+	files := uint64(0)
+
+	for object := range ch {
+		if object.Err != nil {
+			fs.logger.WithError(object.Err).Log("Listing object failed")
+			continue
+		}
+
+		files++
+	}
+
+	return files > 0
+}
+
+func (fs *s3Filesystem) cleanPath(path string) string {
+	if !filepath.IsAbs(path) {
+		path = filepath.Join("/", path)
+	}
+
+	path = strings.TrimSuffix(path, "/"+fakeDirEntry)
+
+	return filepath.Join("/", filepath.Clean(path))[1:]
+}
+
 type s3FileInfo struct {
 	name         string
 	size         int64
+	dir          bool
 	lastModified time.Time
 }
 
@@ -349,6 +572,10 @@ func (f *s3FileInfo) Size() int64 {
 	return f.size
 }
 
+func (f *s3FileInfo) Mode() os.FileMode {
+	return fs.FileMode(fs.ModePerm)
+}
+
 func (f *s3FileInfo) ModTime() time.Time {
 	return f.lastModified
 }
@@ -358,7 +585,7 @@ func (f *s3FileInfo) IsLink() (string, bool) {
 }
 
 func (f *s3FileInfo) IsDir() bool {
-	return false
+	return f.dir
 }
 
 type s3File struct {

io/fs/sized.go

@@ -0,0 +1,168 @@
+package fs
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+)
+
+type SizedFilesystem interface {
+	Filesystem
+
+	// Resize resizes the filesystem to the new size. Files may need to be deleted.
+	Resize(size int64) error
+}
+
+type PurgeFilesystem interface {
+	// Purge will free up at least size number of bytes and returns the actual
+	// freed space in bytes.
+	Purge(size int64) int64
+}
+
+type sizedFilesystem struct {
+	Filesystem
+
+	// Siez is the capacity of the filesystem in bytes
+	maxSize int64
+
+	// Set true to automatically delete the oldest files until there's
+	// enough space to store a new file
+	purge bool
+}
+
+var _ PurgeFilesystem = &sizedFilesystem{}
+
+func NewSizedFilesystem(fs Filesystem, maxSize int64, purge bool) (SizedFilesystem, error) {
+	r := &sizedFilesystem{
+		Filesystem: fs,
+		maxSize:    maxSize,
+		purge:      purge,
+	}
+
+	return r, nil
+}
+
+func (r *sizedFilesystem) Size() (int64, int64) {
+	currentSize, _ := r.Filesystem.Size()
+
+	return currentSize, r.maxSize
+}
+
+func (r *sizedFilesystem) Resize(size int64) error {
+	currentSize, _ := r.Size()
+	if size >= currentSize {
+		// If the new size is the same or larger than the current size,
+		// nothing to do.
+		r.maxSize = size
+		return nil
+	}
+
+	// If the new size is less than the current size, purge some files.
+	r.Purge(currentSize - size)
+
+	r.maxSize = size
+
+	return nil
+}
+
+func (r *sizedFilesystem) WriteFileReader(path string, rd io.Reader) (int64, bool, error) {
+	currentSize, maxSize := r.Size()
+	if maxSize < 0 {
+		return r.Filesystem.WriteFileReader(path, rd)
+	}
+
+	data := bytes.Buffer{}
+	size, err := data.ReadFrom(rd)
+	if err != nil {
+		return -1, false, err
+	}
+
+	// reject if the new file is larger than the available space
+	if size > maxSize {
+		return -1, false, fmt.Errorf("File is too big")
+	}
+
+	// Calculate the new size of the filesystem
+	newSize := currentSize + size
+
+	// If the the new size is larger than the allowed size, we have to free
+	// some space.
+	if newSize > maxSize {
+		if !r.purge {
+			return -1, false, fmt.Errorf("not enough space on device")
+		}
+
+		if r.Purge(size) < size {
+			return -1, false, fmt.Errorf("not enough space on device")
+		}
+	}
+
+	return r.Filesystem.WriteFileReader(path, &data)
+}
+
+func (r *sizedFilesystem) WriteFile(path string, data []byte) (int64, bool, error) {
+	return r.WriteFileReader(path, bytes.NewBuffer(data))
+}
+
+func (r *sizedFilesystem) WriteFileSafe(path string, data []byte) (int64, bool, error) {
+	currentSize, maxSize := r.Size()
+	if maxSize < 0 {
+		return r.Filesystem.WriteFile(path, data)
+	}
+
+	size := int64(len(data))
+
+	// reject if the new file is larger than the available space
+	if size > maxSize {
+		return -1, false, fmt.Errorf("File is too big")
+	}
+
+	// Calculate the new size of the filesystem
+	newSize := currentSize + size
+
+	// If the the new size is larger than the allowed size, we have to free
+	// some space.
+	if newSize > maxSize {
+		if !r.purge {
+			return -1, false, fmt.Errorf("not enough space on device")
+		}
+
+		if r.Purge(size) < size {
+			return -1, false, fmt.Errorf("not enough space on device")
+		}
+	}
+
+	return r.Filesystem.WriteFileSafe(path, data)
+}
+
+func (r *sizedFilesystem) Purge(size int64) int64 {
+	if purger, ok := r.Filesystem.(PurgeFilesystem); ok {
+		return purger.Purge(size)
+	}
+
+	return 0
+	/*
+	   files := r.Filesystem.List("/", "")
+
+	   	sort.Slice(files, func(i, j int) bool {
+	   		return files[i].ModTime().Before(files[j].ModTime())
+	   	})
+
+	   var freed int64 = 0
+
+	   	for _, f := range files {
+	   		r.Filesystem.Remove(f.Name())
+	   		size -= f.Size()
+	   		freed += f.Size()
+	   		r.currentSize -= f.Size()
+
+	   		if size <= 0 {
+	   			break
+	   		}
+	   	}
+
+	   files = nil
+
+	   return freed
+	*/
+}

io/fs/sized_test.go

@@ -0,0 +1,350 @@
+package fs
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func newMemFS() Filesystem {
+	mem, _ := NewMemFilesystem(MemConfig{})
+
+	return mem
+}
+
+func TestNewSized(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(0), cur)
+}
+
+func TestSizedResize(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(10), max)
+
+	err := fs.Resize(20)
+	require.NoError(t, err)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(20), max)
+}
+
+func TestSizedResizePurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(10), max)
+
+	fs.WriteFileReader("/foobar", strings.NewReader("xxxxxxxxxx"))
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(10), cur)
+	require.Equal(t, int64(10), max)
+
+	err := fs.Resize(5)
+	require.NoError(t, err)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(5), max)
+}
+
+func TestSizedWrite(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(0), cur)
+	require.Equal(t, int64(10), max)
+
+	size, created, err := fs.WriteFileReader("/foobar", strings.NewReader("xxxxx"))
+	require.NoError(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(10), max)
+
+	_, _, err = fs.WriteFile("/foobaz", []byte("xxxxxx"))
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileReader("/foobaz", strings.NewReader("xxxxxx"))
+	require.Error(t, err)
+
+	_, _, err = fs.WriteFileSafe("/foobaz", []byte("xxxxxx"))
+	require.Error(t, err)
+}
+
+func TestSizedReplaceNoPurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	data := strings.NewReader("xxxxx")
+
+	size, created, err := fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+
+	data = strings.NewReader("yyy")
+
+	size, created, err = fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(3), size)
+	require.Equal(t, false, created)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(3), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func TestSizedReplacePurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
+
+	data1 := strings.NewReader("xxx")
+	data2 := strings.NewReader("yyy")
+	data3 := strings.NewReader("zzz")
+
+	fs.WriteFileReader("/foobar1", data1)
+	fs.WriteFileReader("/foobar2", data2)
+	fs.WriteFileReader("/foobar3", data3)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(9), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(3), cur)
+
+	data4 := strings.NewReader("zzzzz")
+
+	size, _, _ := fs.WriteFileReader("/foobar1", data4)
+
+	require.Equal(t, int64(5), size)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(8), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(2), cur)
+}
+
+func TestSizedReplaceUnlimited(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), -1, false)
+
+	data := strings.NewReader("xxxxx")
+
+	size, created, err := fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(5), size)
+	require.Equal(t, true, created)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(5), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+
+	data = strings.NewReader("yyy")
+
+	size, created, err = fs.WriteFileReader("/foobar", data)
+
+	require.Nil(t, err)
+	require.Equal(t, int64(3), size)
+	require.Equal(t, false, created)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(3), cur)
+	require.Equal(t, int64(-1), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(1), cur)
+}
+
+func TestSizedTooBigNoPurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	data := strings.NewReader("xxxxxyyyyyz")
+
+	size, _, err := fs.WriteFileReader("/foobar", data)
+	require.Error(t, err)
+	require.Equal(t, int64(-1), size)
+}
+
+func TestSizedTooBigPurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
+
+	data1 := strings.NewReader("xxxxx")
+	data2 := strings.NewReader("yyyyy")
+
+	fs.WriteFileReader("/foobar1", data1)
+	fs.WriteFileReader("/foobar2", data2)
+
+	data := strings.NewReader("xxxxxyyyyyz")
+
+	size, _, err := fs.WriteFileReader("/foobar", data)
+	require.Error(t, err)
+	require.Equal(t, int64(-1), size)
+
+	require.Equal(t, int64(2), fs.Files())
+}
+
+func TestSizedFullSpaceNoPurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, false)
+
+	data1 := strings.NewReader("xxxxx")
+	data2 := strings.NewReader("yyyyy")
+
+	fs.WriteFileReader("/foobar1", data1)
+	fs.WriteFileReader("/foobar2", data2)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(10), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(2), cur)
+
+	data3 := strings.NewReader("zzzzz")
+
+	size, _, err := fs.WriteFileReader("/foobar3", data3)
+	require.Error(t, err)
+	require.Equal(t, int64(-1), size)
+}
+
+func TestSizedFullSpacePurge(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
+
+	data1 := strings.NewReader("xxxxx")
+	data2 := strings.NewReader("yyyyy")
+
+	fs.WriteFileReader("/foobar1", data1)
+	fs.WriteFileReader("/foobar2", data2)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(10), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(2), cur)
+
+	data3 := strings.NewReader("zzzzz")
+
+	size, _, _ := fs.WriteFileReader("/foobar3", data3)
+
+	require.Equal(t, int64(5), size)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(10), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(2), cur)
+}
+
+func TestSizedFullSpacePurgeMulti(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
+
+	data1 := strings.NewReader("xxx")
+	data2 := strings.NewReader("yyy")
+	data3 := strings.NewReader("zzz")
+
+	fs.WriteFileReader("/foobar1", data1)
+	fs.WriteFileReader("/foobar2", data2)
+	fs.WriteFileReader("/foobar3", data3)
+
+	cur, max := fs.Size()
+
+	require.Equal(t, int64(9), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(3), cur)
+
+	data4 := strings.NewReader("zzzzz")
+
+	size, _, _ := fs.WriteFileReader("/foobar4", data4)
+
+	require.Equal(t, int64(5), size)
+
+	cur, max = fs.Size()
+
+	require.Equal(t, int64(8), cur)
+	require.Equal(t, int64(10), max)
+
+	cur = fs.Files()
+
+	require.Equal(t, int64(2), cur)
+}
+
+func TestSizedPurgeOrder(t *testing.T) {
+	fs, _ := NewSizedFilesystem(newMemFS(), 10, true)
+
+	data1 := strings.NewReader("xxxxx")
+	data2 := strings.NewReader("yyyyy")
+	data3 := strings.NewReader("zzzzz")
+
+	fs.WriteFileReader("/foobar1", data1)
+	time.Sleep(1 * time.Second)
+	fs.WriteFileReader("/foobar2", data2)
+	time.Sleep(1 * time.Second)
+	fs.WriteFileReader("/foobar3", data3)
+
+	file := fs.Open("/foobar1")
+
+	require.Nil(t, file)
+}

restream/fs/fs.go

@@ -135,7 +135,7 @@ func (rfs *filesystem) cleanup() {
 
 	for _, patterns := range rfs.cleanupPatterns {
 		for _, pattern := range patterns {
-			filesAndDirs := rfs.Filesystem.List(pattern.Pattern)
+			filesAndDirs := rfs.Filesystem.List("/", pattern.Pattern)
 
 			files := []fs.FileInfo{}
 			for _, f := range filesAndDirs {
@@ -151,7 +151,7 @@ func (rfs *filesystem) cleanup() {
 			if pattern.MaxFiles > 0 && uint(len(files)) > pattern.MaxFiles {
 				for i := uint(0); i < uint(len(files))-pattern.MaxFiles; i++ {
 					rfs.logger.Debug().WithField("path", files[i].Name()).Log("Remove file because MaxFiles is exceeded")
-					rfs.Filesystem.Delete(files[i].Name())
+					rfs.Filesystem.Remove(files[i].Name())
 				}
 			}
 
@@ -161,7 +161,7 @@ func (rfs *filesystem) cleanup() {
 				for _, f := range files {
 					if f.ModTime().Before(bestBefore) {
 						rfs.logger.Debug().WithField("path", f.Name()).Log("Remove file because MaxFileAge is exceeded")
-						rfs.Filesystem.Delete(f.Name())
+						rfs.Filesystem.Remove(f.Name())
 					}
 				}
 			}
@@ -175,11 +175,11 @@ func (rfs *filesystem) purge(patterns []Pattern) (nfiles uint64) {
 			continue
 		}
 
-		files := rfs.Filesystem.List(pattern.Pattern)
+		files := rfs.Filesystem.List("/", pattern.Pattern)
 		sort.Slice(files, func(i, j int) bool { return len(files[i].Name()) > len(files[j].Name()) })
 		for _, f := range files {
 			rfs.logger.Debug().WithField("path", f.Name()).Log("Purging file")
-			rfs.Filesystem.Delete(f.Name())
+			rfs.Filesystem.Remove(f.Name())
 			nfiles++
 		}
 	}

restream/fs/fs_test.go

@@ -10,11 +10,7 @@ import (
 )
 
 func TestMaxFiles(t *testing.T) {
-	memfs := fs.NewMemFilesystem(fs.MemConfig{
-		Base:  "/",
-		Size:  1024,
-		Purge: false,
-	})
+	memfs, _ := fs.NewMemFilesystem(fs.MemConfig{})
 
 	cleanfs := New(Config{
 		FS: memfs,
@@ -30,15 +26,15 @@ func TestMaxFiles(t *testing.T) {
 		},
 	})
 
-	cleanfs.Store("/chunk_0.ts", strings.NewReader("chunk_0"))
-	cleanfs.Store("/chunk_1.ts", strings.NewReader("chunk_1"))
-	cleanfs.Store("/chunk_2.ts", strings.NewReader("chunk_2"))
+	cleanfs.WriteFileReader("/chunk_0.ts", strings.NewReader("chunk_0"))
+	cleanfs.WriteFileReader("/chunk_1.ts", strings.NewReader("chunk_1"))
+	cleanfs.WriteFileReader("/chunk_2.ts", strings.NewReader("chunk_2"))
 
 	require.Eventually(t, func() bool {
 		return cleanfs.Files() == 3
 	}, 3*time.Second, time.Second)
 
-	cleanfs.Store("/chunk_3.ts", strings.NewReader("chunk_3"))
+	cleanfs.WriteFileReader("/chunk_3.ts", strings.NewReader("chunk_3"))
 
 	require.Eventually(t, func() bool {
 		if cleanfs.Files() != 3 {
@@ -47,7 +43,7 @@ func TestMaxFiles(t *testing.T) {
 
 		names := []string{}
 
-		for _, f := range cleanfs.List("/*.ts") {
+		for _, f := range cleanfs.List("/", "/*.ts") {
 			names = append(names, f.Name())
 		}
 
@@ -60,11 +56,7 @@ func TestMaxFiles(t *testing.T) {
 }
 
 func TestMaxAge(t *testing.T) {
-	memfs := fs.NewMemFilesystem(fs.MemConfig{
-		Base:  "/",
-		Size:  1024,
-		Purge: false,
-	})
+	memfs, _ := fs.NewMemFilesystem(fs.MemConfig{})
 
 	cleanfs := New(Config{
 		FS: memfs,
@@ -80,15 +72,15 @@ func TestMaxAge(t *testing.T) {
 		},
 	})
 
-	cleanfs.Store("/chunk_0.ts", strings.NewReader("chunk_0"))
-	cleanfs.Store("/chunk_1.ts", strings.NewReader("chunk_1"))
-	cleanfs.Store("/chunk_2.ts", strings.NewReader("chunk_2"))
+	cleanfs.WriteFileReader("/chunk_0.ts", strings.NewReader("chunk_0"))
+	cleanfs.WriteFileReader("/chunk_1.ts", strings.NewReader("chunk_1"))
+	cleanfs.WriteFileReader("/chunk_2.ts", strings.NewReader("chunk_2"))
 
 	require.Eventually(t, func() bool {
 		return cleanfs.Files() == 0
 	}, 5*time.Second, time.Second)
 
-	cleanfs.Store("/chunk_3.ts", strings.NewReader("chunk_3"))
+	cleanfs.WriteFileReader("/chunk_3.ts", strings.NewReader("chunk_3"))
 
 	require.Eventually(t, func() bool {
 		if cleanfs.Files() != 1 {
@@ -97,7 +89,7 @@ func TestMaxAge(t *testing.T) {
 
 		names := []string{}
 
-		for _, f := range cleanfs.List("/*.ts") {
+		for _, f := range cleanfs.List("/", "/*.ts") {
 			names = append(names, f.Name())
 		}
 
@@ -110,11 +102,7 @@ func TestMaxAge(t *testing.T) {
 }
 
 func TestUnsetCleanup(t *testing.T) {
-	memfs := fs.NewMemFilesystem(fs.MemConfig{
-		Base:  "/",
-		Size:  1024,
-		Purge: false,
-	})
+	memfs, _ := fs.NewMemFilesystem(fs.MemConfig{})
 
 	cleanfs := New(Config{
 		FS: memfs,
@@ -130,15 +118,15 @@ func TestUnsetCleanup(t *testing.T) {
 		},
 	})
 
-	cleanfs.Store("/chunk_0.ts", strings.NewReader("chunk_0"))
-	cleanfs.Store("/chunk_1.ts", strings.NewReader("chunk_1"))
-	cleanfs.Store("/chunk_2.ts", strings.NewReader("chunk_2"))
+	cleanfs.WriteFileReader("/chunk_0.ts", strings.NewReader("chunk_0"))
+	cleanfs.WriteFileReader("/chunk_1.ts", strings.NewReader("chunk_1"))
+	cleanfs.WriteFileReader("/chunk_2.ts", strings.NewReader("chunk_2"))
 
 	require.Eventually(t, func() bool {
 		return cleanfs.Files() == 3
 	}, 3*time.Second, time.Second)
 
-	cleanfs.Store("/chunk_3.ts", strings.NewReader("chunk_3"))
+	cleanfs.WriteFileReader("/chunk_3.ts", strings.NewReader("chunk_3"))
 
 	require.Eventually(t, func() bool {
 		if cleanfs.Files() != 3 {
@@ -147,7 +135,7 @@ func TestUnsetCleanup(t *testing.T) {
 
 		names := []string{}
 
-		for _, f := range cleanfs.List("/*.ts") {
+		for _, f := range cleanfs.List("/", "/*.ts") {
 			names = append(names, f.Name())
 		}
 
@@ -158,7 +146,7 @@ func TestUnsetCleanup(t *testing.T) {
 
 	cleanfs.UnsetCleanup("foobar")
 
-	cleanfs.Store("/chunk_4.ts", strings.NewReader("chunk_4"))
+	cleanfs.WriteFileReader("/chunk_4.ts", strings.NewReader("chunk_4"))
 
 	require.Eventually(t, func() bool {
 		if cleanfs.Files() != 4 {
@@ -167,7 +155,7 @@ func TestUnsetCleanup(t *testing.T) {
 
 		names := []string{}
 
-		for _, f := range cleanfs.List("/*.ts") {
+		for _, f := range cleanfs.List("/", "/*.ts") {
 			names = append(names, f.Name())
 		}
 

restream/restream.go

@@ -124,7 +124,14 @@ func New(config Config) (Restreamer, error) {
 	}
 
 	if r.store == nil {
-		r.store = store.NewDummyStore(store.DummyConfig{})
+		dummyfs, _ := fs.NewMemFilesystem(fs.MemConfig{})
+		s, err := store.NewJSON(store.JSONConfig{
+			Filesystem: dummyfs,
+		})
+		if err != nil {
+			return nil, err
+		}
+		r.store = s
 	}
 
 	for _, fs := range config.Filesystems {
@@ -136,7 +143,7 @@ func New(config Config) (Restreamer, error) {
 		r.fs.list = append(r.fs.list, fs)
 
 		// Add the diskfs filesystems also to a separate array. We need it later for input and output validation
-		if fs.Type() == "diskfs" {
+		if fs.Type() == "disk" {
 			r.fs.diskfs = append(r.fs.diskfs, fs)
 		}
 	}
@@ -183,7 +190,7 @@ func (r *restream) Start() {
 		for _, fs := range r.fs.list {
 			fs.Start()
 
-			if fs.Type() == "diskfs" {
+			if fs.Type() == "disk" {
 				go r.observe(ctx, fs, 10*time.Second)
 			}
 		}
@@ -635,7 +642,7 @@ func (r *restream) validateConfig(config *app.Config) (bool, error) {
 		if len(r.fs.diskfs) != 0 {
 			maxFails := 0
 			for _, fs := range r.fs.diskfs {
-				io.Address, err = r.validateInputAddress(io.Address, fs.Base())
+				io.Address, err = r.validateInputAddress(io.Address, fs.Metadata("base"))
 				if err != nil {
 					maxFails++
 				}
@@ -682,7 +689,7 @@ func (r *restream) validateConfig(config *app.Config) (bool, error) {
 			maxFails := 0
 			for _, fs := range r.fs.diskfs {
 				isFile := false
-				io.Address, isFile, err = r.validateOutputAddress(io.Address, fs.Base())
+				io.Address, isFile, err = r.validateOutputAddress(io.Address, fs.Metadata("base"))
 				if err != nil {
 					maxFails++
 				}

restream/store/dummy.go

@@ -1,37 +0,0 @@
-package store
-
-import (
-	"github.com/datarhei/core/v16/log"
-)
-
-type DummyConfig struct {
-	Logger log.Logger
-}
-
-type dummyStore struct {
-	logger log.Logger
-}
-
-func NewDummyStore(config DummyConfig) Store {
-	s := &dummyStore{
-		logger: config.Logger,
-	}
-
-	if s.logger == nil {
-		s.logger = log.New("")
-	}
-
-	return s
-}
-
-func (sb *dummyStore) Store(data StoreData) error {
-	sb.logger.Debug().Log("Data stored")
-
-	return nil
-}
-
-func (sb *dummyStore) Load() (StoreData, error) {
-	sb.logger.Debug().Log("Data loaded")
-
-	return NewStoreData(), nil
-}

restream/store/json.go

@@ -4,24 +4,23 @@ import (
 	gojson "encoding/json"
 	"fmt"
 	"os"
-	"path"
 	"sync"
 
 	"github.com/datarhei/core/v16/encoding/json"
-	"github.com/datarhei/core/v16/io/file"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/log"
 )
 
 type JSONConfig struct {
-	Filepath  string
-	FFVersion string
-	Logger    log.Logger
+	Filesystem fs.Filesystem
+	Filepath   string // Full path to the database file
+	Logger     log.Logger
 }
 
 type jsonStore struct {
-	filepath  string
-	ffversion string
-	logger    log.Logger
+	fs       fs.Filesystem
+	filepath string
+	logger   log.Logger
 
 	// Mutex to serialize access to the backend
 	lock sync.RWMutex
@@ -29,18 +28,26 @@ type jsonStore struct {
 
 var version uint64 = 4
 
-func NewJSONStore(config JSONConfig) Store {
+func NewJSON(config JSONConfig) (Store, error) {
 	s := &jsonStore{
-		filepath:  config.Filepath,
-		ffversion: config.FFVersion,
-		logger:    config.Logger,
+		fs:       config.Filesystem,
+		filepath: config.Filepath,
+		logger:   config.Logger,
+	}
+
+	if len(s.filepath) == 0 {
+		s.filepath = "/db.json"
+	}
+
+	if s.fs == nil {
+		return nil, fmt.Errorf("no valid filesystem provided")
 	}
 
 	if s.logger == nil {
 		s.logger = log.New("")
 	}
 
-	return s
+	return s, nil
 }
 
 func (s *jsonStore) Load() (StoreData, error) {
@@ -79,28 +86,11 @@ func (s *jsonStore) store(filepath string, data StoreData) error {
 		return err
 	}
 
-	dir := path.Dir(filepath)
-	name := path.Base(filepath)
-
-	tmpfile, err := os.CreateTemp(dir, name)
+	_, _, err = s.fs.WriteFileSafe(filepath, jsondata)
 	if err != nil {
 		return err
 	}
 
-	defer os.Remove(tmpfile.Name())
-
-	if _, err := tmpfile.Write(jsondata); err != nil {
-		return err
-	}
-
-	if err := tmpfile.Close(); err != nil {
-		return err
-	}
-
-	if err := file.Rename(tmpfile.Name(), filepath); err != nil {
-		return err
-	}
-
 	s.logger.WithField("file", filepath).Debug().Log("Stored data")
 
 	return nil
@@ -113,7 +103,7 @@ type storeVersion struct {
 func (s *jsonStore) load(filepath string, version uint64) (StoreData, error) {
 	r := NewStoreData()
 
-	_, err := os.Stat(filepath)
+	_, err := s.fs.Stat(filepath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return r, nil
@@ -122,7 +112,7 @@ func (s *jsonStore) load(filepath string, version uint64) (StoreData, error) {
 		return r, err
 	}
 
-	jsondata, err := os.ReadFile(filepath)
+	jsondata, err := s.fs.ReadFile(filepath)
 	if err != nil {
 		return r, err
 	}

restream/store/json_test.go

@@ -1,40 +1,61 @@
 package store
 
 import (
-	"os"
 	"testing"
 
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/stretchr/testify/require"
 )
 
-func TestNew(t *testing.T) {
-	store := NewJSONStore(JSONConfig{})
+func getFS(t *testing.T) fs.Filesystem {
+	fs, err := fs.NewRootedDiskFilesystem(fs.RootedDiskConfig{
+		Root: ".",
+	})
+	require.NoError(t, err)
 
+	info, err := fs.Stat("./fixtures/v4_empty.json")
+	require.NoError(t, err)
+	require.Equal(t, "/fixtures/v4_empty.json", info.Name())
+
+	return fs
+}
+
+func TestNew(t *testing.T) {
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+	})
+	require.NoError(t, err)
 	require.NotEmpty(t, store)
 }
 
 func TestLoad(t *testing.T) {
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v4_empty.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+		Filepath:   "./fixtures/v4_empty.json",
 	})
+	require.NoError(t, err)
 
-	_, err := store.Load()
-	require.Equal(t, nil, err)
+	_, err = store.Load()
+	require.NoError(t, err)
 }
 
 func TestLoadFailed(t *testing.T) {
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v4_invalid.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+		Filepath:   "./fixtures/v4_invalid.json",
 	})
+	require.NoError(t, err)
 
-	_, err := store.Load()
-	require.NotEqual(t, nil, err)
+	_, err = store.Load()
+	require.Error(t, err)
 }
 
 func TestIsEmpty(t *testing.T) {
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v4_empty.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+		Filepath:   "./fixtures/v4_empty.json",
 	})
+	require.NoError(t, err)
 
 	data, err := store.Load()
 	require.NoError(t, err)
@@ -42,9 +63,11 @@ func TestIsEmpty(t *testing.T) {
 }
 
 func TestNotExists(t *testing.T) {
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v4_notexist.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+		Filepath:   "./fixtures/v4_notexist.json",
 	})
+	require.NoError(t, err)
 
 	data, err := store.Load()
 	require.NoError(t, err)
@@ -52,11 +75,14 @@ func TestNotExists(t *testing.T) {
 }
 
 func TestStore(t *testing.T) {
-	os.Remove("./fixtures/v4_store.json")
+	fs := getFS(t)
+	fs.Remove("./fixtures/v4_store.json")
 
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v4_store.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: fs,
+		Filepath:   "./fixtures/v4_store.json",
 	})
+	require.NoError(t, err)
 
 	data, err := store.Load()
 	require.NoError(t, err)
@@ -70,13 +96,15 @@ func TestStore(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, data, data2)
 
-	os.Remove("./fixtures/v4_store.json")
+	fs.Remove("./fixtures/v4_store.json")
 }
 
 func TestInvalidVersion(t *testing.T) {
-	store := NewJSONStore(JSONConfig{
-		Filepath: "./fixtures/v3_empty.json",
+	store, err := NewJSON(JSONConfig{
+		Filesystem: getFS(t),
+		Filepath:   "./fixtures/v3_empty.json",
 	})
+	require.NoError(t, err)
 
 	data, err := store.Load()
 	require.Error(t, err)

session/collector.go

@@ -3,13 +3,11 @@ package session
 import (
 	"context"
 	"encoding/json"
-	"os"
-	"path/filepath"
 	"sort"
 	"sync"
 	"time"
 
-	"github.com/datarhei/core/v16/io/file"
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/log"
 	"github.com/datarhei/core/v16/net"
 
@@ -244,6 +242,7 @@ type collector struct {
 
 	persist struct {
 		enable   bool
+		fs       fs.Filesystem
 		path     string
 		interval time.Duration
 		done     context.CancelFunc
@@ -275,7 +274,7 @@ const (
 // NewCollector returns a new collector according to the provided configuration. If such a
 // collector can't be created, a NullCollector is returned.
 func NewCollector(config CollectorConfig) Collector {
-	collector, err := newCollector("", "", nil, config)
+	collector, err := newCollector("", nil, nil, config)
 	if err != nil {
 		return NewNullCollector()
 	}
@@ -285,7 +284,7 @@ func NewCollector(config CollectorConfig) Collector {
 	return collector
 }
 
-func newCollector(id, persistPath string, logger log.Logger, config CollectorConfig) (*collector, error) {
+func newCollector(id string, persistFS fs.Filesystem, logger log.Logger, config CollectorConfig) (*collector, error) {
 	c := &collector{
 		maxRxBitrate:    float64(config.MaxRxBitrate),
 		maxTxBitrate:    float64(config.MaxTxBitrate),
@@ -379,11 +378,12 @@ func newCollector(id, persistPath string, logger log.Logger, config CollectorCon
 
 	c.history.Sessions = make(map[string]totals)
 
-	c.persist.enable = len(persistPath) != 0
-	c.persist.path = persistPath
+	c.persist.enable = persistFS != nil
+	c.persist.fs = persistFS
+	c.persist.path = "/" + id + ".json"
 	c.persist.interval = config.PersistInterval
 
-	c.loadHistory(c.persist.path, &c.history)
+	c.loadHistory(c.persist.fs, c.persist.path, &c.history)
 
 	c.stopOnce.Do(func() {})
 
@@ -433,7 +433,7 @@ func (c *collector) Persist() {
 	c.lock.history.RLock()
 	defer c.lock.history.RUnlock()
 
-	c.saveHistory(c.persist.path, &c.history)
+	c.saveHistory(c.persist.fs, c.persist.path, &c.history)
 }
 
 func (c *collector) persister(ctx context.Context, interval time.Duration) {
@@ -450,17 +450,20 @@ func (c *collector) persister(ctx context.Context, interval time.Duration) {
 	}
 }
 
-func (c *collector) loadHistory(path string, data *history) {
-	c.logger.WithComponent("SessionStore").WithField("path", path).Debug().Log("Loading history")
-
-	if len(path) == 0 {
+func (c *collector) loadHistory(fs fs.Filesystem, path string, data *history) {
+	if fs == nil {
 		return
 	}
 
+	c.logger.WithComponent("SessionStore").WithFields(log.Fields{
+		"base": fs.Metadata("base"),
+		"path": path,
+	}).Debug().Log("Loading history")
+
 	c.lock.persist.Lock()
 	defer c.lock.persist.Unlock()
 
-	jsondata, err := os.ReadFile(path)
+	jsondata, err := fs.ReadFile(path)
 	if err != nil {
 		return
 	}
@@ -470,12 +473,15 @@ func (c *collector) loadHistory(path string, data *history) {
 	}
 }
 
-func (c *collector) saveHistory(path string, data *history) {
-	if len(path) == 0 {
+func (c *collector) saveHistory(fs fs.Filesystem, path string, data *history) {
+	if fs == nil {
 		return
 	}
 
-	c.logger.WithComponent("SessionStore").WithField("path", path).Debug().Log("Storing history")
+	c.logger.WithComponent("SessionStore").WithFields(log.Fields{
+		"base": fs.Metadata("base"),
+		"path": path,
+	}).Debug().Log("Storing history")
 
 	c.lock.persist.Lock()
 	defer c.lock.persist.Unlock()
@@ -485,27 +491,10 @@ func (c *collector) saveHistory(path string, data *history) {
 		return
 	}
 
-	dir := filepath.Dir(path)
-	filename := filepath.Base(path)
-
-	tmpfile, err := os.CreateTemp(dir, filename)
+	_, _, err = fs.WriteFileSafe(path, jsondata)
 	if err != nil {
 		return
 	}
-
-	defer os.Remove(tmpfile.Name())
-
-	if _, err := tmpfile.Write(jsondata); err != nil {
-		return
-	}
-
-	if err := tmpfile.Close(); err != nil {
-		return
-	}
-
-	if err := file.Rename(tmpfile.Name(), path); err != nil {
-		return
-	}
 }
 
 func (c *collector) IsCollectableIP(ip string) bool {

session/collector_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestRegisterSession(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Hour,
 		SessionTimeout:  time.Hour,
 	})
@@ -31,7 +31,7 @@ func TestRegisterSession(t *testing.T) {
 }
 
 func TestInactiveSession(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Second,
 		SessionTimeout:  time.Hour,
 	})
@@ -52,7 +52,7 @@ func TestInactiveSession(t *testing.T) {
 }
 
 func TestActivateSession(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Second,
 		SessionTimeout:  time.Second,
 	})
@@ -73,7 +73,7 @@ func TestActivateSession(t *testing.T) {
 }
 
 func TestIngress(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Second,
 		SessionTimeout:  time.Hour,
 	})
@@ -92,7 +92,7 @@ func TestIngress(t *testing.T) {
 }
 
 func TestEgress(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Second,
 		SessionTimeout:  time.Hour,
 	})
@@ -111,7 +111,7 @@ func TestEgress(t *testing.T) {
 }
 
 func TestNbSessions(t *testing.T) {
-	c, err := newCollector("", "", nil, CollectorConfig{
+	c, err := newCollector("", nil, nil, CollectorConfig{
 		InactiveTimeout: time.Hour,
 		SessionTimeout:  time.Hour,
 	})

session/registry.go

@@ -2,20 +2,18 @@ package session
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"regexp"
 	"sync"
 
+	"github.com/datarhei/core/v16/io/fs"
 	"github.com/datarhei/core/v16/log"
 )
 
 // Config is the configuration for creating a new registry
 type Config struct {
-	// PersistDir is a path to the directory where the session
-	// history will be persisted. If it is an empty value, the
+	// PersistFS is a filesystem in whose root the session history will be persisted. If it is nil, the
 	// history will not be persisted.
-	PersistDir string
+	PersistFS fs.Filesystem
 
 	// Logger is an instance of a logger. If it is nil, no logs
 	// will be written.
@@ -52,9 +50,9 @@ type Registry interface {
 }
 
 type registry struct {
-	collector  map[string]*collector
-	persistDir string
-	logger     log.Logger
+	collector map[string]*collector
+	persistFS fs.Filesystem
+	logger    log.Logger
 
 	lock sync.Mutex
 }
@@ -63,21 +61,15 @@ type registry struct {
 // is non-nil if the PersistDir from the config can't be created.
 func New(conf Config) (Registry, error) {
 	r := &registry{
-		collector:  make(map[string]*collector),
-		persistDir: conf.PersistDir,
-		logger:     conf.Logger,
+		collector: make(map[string]*collector),
+		persistFS: conf.PersistFS,
+		logger:    conf.Logger,
 	}
 
 	if r.logger == nil {
 		r.logger = log.New("Session")
 	}
 
-	if len(r.persistDir) != 0 {
-		if err := os.MkdirAll(r.persistDir, 0700); err != nil {
-			return nil, err
-		}
-	}
-
 	return r, nil
 }
 
@@ -99,12 +91,7 @@ func (r *registry) Register(id string, conf CollectorConfig) (Collector, error)
 		return nil, fmt.Errorf("a collector with the ID '%s' already exists", id)
 	}
 
-	persistPath := ""
-	if len(r.persistDir) != 0 {
-		persistPath = filepath.Join(r.persistDir, id+".json")
-	}
-
-	m, err := newCollector(id, persistPath, r.logger, conf)
+	m, err := newCollector(id, r.persistFS, r.logger, conf)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/cespare/xxhash/v2/README.md

@@ -3,8 +3,7 @@
 [![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2)
 [![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml)
 
-xxhash is a Go implementation of the 64-bit
-[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
+xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a
 high-quality hashing algorithm that is much faster than anything in the Go
 standard library.
 
@@ -25,8 +24,11 @@ func (*Digest) WriteString(string) (int, error)
 func (*Digest) Sum64() uint64
 ```
 
-This implementation provides a fast pure-Go implementation and an even faster
-assembly implementation for amd64.
+The package is written with optimized pure Go and also contains even faster
+assembly implementations for amd64 and arm64. If desired, the `purego` build tag
+opts into using the Go code even on those architectures.
+
+[xxHash]: http://cyan4973.github.io/xxHash/
 
 ## Compatibility
 
@@ -45,19 +47,20 @@ I recommend using the latest release of Go.
 Here are some quick benchmarks comparing the pure-Go and assembly
 implementations of Sum64.
 
-| input size | purego | asm |
-| --- | --- | --- |
-| 5 B   |  979.66 MB/s |  1291.17 MB/s  |
-| 100 B | 7475.26 MB/s | 7973.40 MB/s  |
-| 4 KB  | 17573.46 MB/s | 17602.65 MB/s |
-| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
+| input size | purego    | asm       |
+| ---------- | --------- | --------- |
+| 4 B        |  1.3 GB/s |  1.2 GB/s |
+| 16 B       |  2.9 GB/s |  3.5 GB/s |
+| 100 B      |  6.9 GB/s |  8.1 GB/s |
+| 4 KB       | 11.7 GB/s | 16.7 GB/s |
+| 10 MB      | 12.0 GB/s | 17.3 GB/s |
 
-These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
-the following commands under Go 1.11.2:
+These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C
+CPU using the following commands under Go 1.19.2:
 
 ```
-$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
-$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
+benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$')
+benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$')
 ```
 
 ## Projects using this package

vendor/github.com/cespare/xxhash/v2/testall.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eu -o pipefail
+
+# Small convenience script for running the tests with various combinations of
+# arch/tags. This assumes we're running on amd64 and have qemu available.
+
+go test ./...
+go test -tags purego ./...
+GOARCH=arm64 go test
+GOARCH=arm64 go test -tags purego

vendor/github.com/cespare/xxhash/v2/xxhash.go

@@ -16,19 +16,11 @@ const (
 	prime5 uint64 = 2870177450012600261
 )
 
-// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
-// possible in the Go code is worth a small (but measurable) performance boost
-// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
-// convenience in the Go code in a few places where we need to intentionally
-// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
-// result overflows a uint64).
-var (
-	prime1v = prime1
-	prime2v = prime2
-	prime3v = prime3
-	prime4v = prime4
-	prime5v = prime5
-)
+// Store the primes in an array as well.
+//
+// The consts are used when possible in Go code to avoid MOVs but we need a
+// contiguous array of the assembly code.
+var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5}
 
 // Digest implements hash.Hash64.
 type Digest struct {
@@ -50,10 +42,10 @@ func New() *Digest {
 
 // Reset clears the Digest's state so that it can be reused.
 func (d *Digest) Reset() {
-	d.v1 = prime1v + prime2
+	d.v1 = primes[0] + prime2
 	d.v2 = prime2
 	d.v3 = 0
-	d.v4 = -prime1v
+	d.v4 = -primes[0]
 	d.total = 0
 	d.n = 0
 }
@@ -69,21 +61,23 @@ func (d *Digest) Write(b []byte) (n int, err error) {
 	n = len(b)
 	d.total += uint64(n)
 
+	memleft := d.mem[d.n&(len(d.mem)-1):]
+
 	if d.n+n < 32 {
 		// This new data doesn't even fill the current block.
-		copy(d.mem[d.n:], b)
+		copy(memleft, b)
 		d.n += n
 		return
 	}
 
 	if d.n > 0 {
 		// Finish off the partial block.
-		copy(d.mem[d.n:], b)
+		c := copy(memleft, b)
 		d.v1 = round(d.v1, u64(d.mem[0:8]))
 		d.v2 = round(d.v2, u64(d.mem[8:16]))
 		d.v3 = round(d.v3, u64(d.mem[16:24]))
 		d.v4 = round(d.v4, u64(d.mem[24:32]))
-		b = b[32-d.n:]
+		b = b[c:]
 		d.n = 0
 	}
 
@@ -133,21 +127,20 @@ func (d *Digest) Sum64() uint64 {
 
 	h += d.total
 
-	i, end := 0, d.n
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(d.mem[i:i+8]))
+	b := d.mem[:d.n&(len(d.mem)-1)]
+	for ; len(b) >= 8; b = b[8:] {
+		k1 := round(0, u64(b[:8]))
 		h ^= k1
 		h = rol27(h)*prime1 + prime4
 	}
-	if i+4 <= end {
-		h ^= uint64(u32(d.mem[i:i+4])) * prime1
+	if len(b) >= 4 {
+		h ^= uint64(u32(b[:4])) * prime1
 		h = rol23(h)*prime2 + prime3
-		i += 4
+		b = b[4:]
 	}
-	for i < end {
-		h ^= uint64(d.mem[i]) * prime5
+	for ; len(b) > 0; b = b[1:] {
+		h ^= uint64(b[0]) * prime5
 		h = rol11(h) * prime1
-		i++
 	}
 
 	h ^= h >> 33

vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s

@@ -1,215 +1,209 @@
+//go:build !appengine && gc && !purego
 // +build !appengine
 // +build gc
 // +build !purego
 
 #include "textflag.h"
 
-// Register allocation:
-// AX	h
-// SI	pointer to advance through b
-// DX	n
-// BX	loop end
-// R8	v1, k1
-// R9	v2
-// R10	v3
-// R11	v4
-// R12	tmp
-// R13	prime1v
-// R14	prime2v
-// DI	prime4v
+// Registers:
+#define h      AX
+#define d      AX
+#define p      SI // pointer to advance through b
+#define n      DX
+#define end    BX // loop end
+#define v1     R8
+#define v2     R9
+#define v3     R10
+#define v4     R11
+#define x      R12
+#define prime1 R13
+#define prime2 R14
+#define prime4 DI
 
-// round reads from and advances the buffer pointer in SI.
-// It assumes that R13 has prime1v and R14 has prime2v.
-#define round(r) \
-	MOVQ  (SI), R12 \
-	ADDQ  $8, SI    \
-	IMULQ R14, R12  \
-	ADDQ  R12, r    \
-	ROLQ  $31, r    \
-	IMULQ R13, r
+#define round(acc, x) \
+	IMULQ prime2, x   \
+	ADDQ  x, acc      \
+	ROLQ  $31, acc    \
+	IMULQ prime1, acc
 
-// mergeRound applies a merge round on the two registers acc and val.
-// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v.
-#define mergeRound(acc, val) \
-	IMULQ R14, val \
-	ROLQ  $31, val \
-	IMULQ R13, val \
-	XORQ  val, acc \
-	IMULQ R13, acc \
-	ADDQ  DI, acc
+// round0 performs the operation x = round(0, x).
+#define round0(x) \
+	IMULQ prime2, x \
+	ROLQ  $31, x    \
+	IMULQ prime1, x
+
+// mergeRound applies a merge round on the two registers acc and x.
+// It assumes that prime1, prime2, and prime4 have been loaded.
+#define mergeRound(acc, x) \
+	round0(x)         \
+	XORQ  x, acc      \
+	IMULQ prime1, acc \
+	ADDQ  prime4, acc
+
+// blockLoop processes as many 32-byte blocks as possible,
+// updating v1, v2, v3, and v4. It assumes that there is at least one block
+// to process.
+#define blockLoop() \
+loop:  \
+	MOVQ +0(p), x  \
+	round(v1, x)   \
+	MOVQ +8(p), x  \
+	round(v2, x)   \
+	MOVQ +16(p), x \
+	round(v3, x)   \
+	MOVQ +24(p), x \
+	round(v4, x)   \
+	ADDQ $32, p    \
+	CMPQ p, end    \
+	JLE  loop
 
 // func Sum64(b []byte) uint64
-TEXT ·Sum64(SB), NOSPLIT, $0-32
+TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
 	// Load fixed primes.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
-	MOVQ ·prime4v(SB), DI
+	MOVQ ·primes+0(SB), prime1
+	MOVQ ·primes+8(SB), prime2
+	MOVQ ·primes+24(SB), prime4
 
 	// Load slice.
-	MOVQ b_base+0(FP), SI
-	MOVQ b_len+8(FP), DX
-	LEAQ (SI)(DX*1), BX
+	MOVQ b_base+0(FP), p
+	MOVQ b_len+8(FP), n
+	LEAQ (p)(n*1), end
 
 	// The first loop limit will be len(b)-32.
-	SUBQ $32, BX
+	SUBQ $32, end
 
 	// Check whether we have at least one block.
-	CMPQ DX, $32
+	CMPQ n, $32
 	JLT  noBlocks
 
 	// Set up initial state (v1, v2, v3, v4).
-	MOVQ R13, R8
-	ADDQ R14, R8
-	MOVQ R14, R9
-	XORQ R10, R10
-	XORQ R11, R11
-	SUBQ R13, R11
+	MOVQ prime1, v1
+	ADDQ prime2, v1
+	MOVQ prime2, v2
+	XORQ v3, v3
+	XORQ v4, v4
+	SUBQ prime1, v4
 
-	// Loop until SI > BX.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
+	blockLoop()
 
-	CMPQ SI, BX
-	JLE  blockLoop
+	MOVQ v1, h
+	ROLQ $1, h
+	MOVQ v2, x
+	ROLQ $7, x
+	ADDQ x, h
+	MOVQ v3, x
+	ROLQ $12, x
+	ADDQ x, h
+	MOVQ v4, x
+	ROLQ $18, x
+	ADDQ x, h
 
-	MOVQ R8, AX
-	ROLQ $1, AX
-	MOVQ R9, R12
-	ROLQ $7, R12
-	ADDQ R12, AX
-	MOVQ R10, R12
-	ROLQ $12, R12
-	ADDQ R12, AX
-	MOVQ R11, R12
-	ROLQ $18, R12
-	ADDQ R12, AX
-
-	mergeRound(AX, R8)
-	mergeRound(AX, R9)
-	mergeRound(AX, R10)
-	mergeRound(AX, R11)
+	mergeRound(h, v1)
+	mergeRound(h, v2)
+	mergeRound(h, v3)
+	mergeRound(h, v4)
 
 	JMP afterBlocks
 
 noBlocks:
-	MOVQ ·prime5v(SB), AX
+	MOVQ ·primes+32(SB), h
 
 afterBlocks:
-	ADDQ DX, AX
+	ADDQ n, h
 
-	// Right now BX has len(b)-32, and we want to loop until SI > len(b)-8.
-	ADDQ $24, BX
+	ADDQ $24, end
+	CMPQ p, end
+	JG   try4
 
-	CMPQ SI, BX
-	JG   fourByte
+loop8:
+	MOVQ  (p), x
+	ADDQ  $8, p
+	round0(x)
+	XORQ  x, h
+	ROLQ  $27, h
+	IMULQ prime1, h
+	ADDQ  prime4, h
 
-wordLoop:
-	// Calculate k1.
-	MOVQ  (SI), R8
-	ADDQ  $8, SI
-	IMULQ R14, R8
-	ROLQ  $31, R8
-	IMULQ R13, R8
+	CMPQ p, end
+	JLE  loop8
 
-	XORQ  R8, AX
-	ROLQ  $27, AX
-	IMULQ R13, AX
-	ADDQ  DI, AX
+try4:
+	ADDQ $4, end
+	CMPQ p, end
+	JG   try1
 
-	CMPQ SI, BX
-	JLE  wordLoop
+	MOVL  (p), x
+	ADDQ  $4, p
+	IMULQ prime1, x
+	XORQ  x, h
 
-fourByte:
-	ADDQ $4, BX
-	CMPQ SI, BX
-	JG   singles
+	ROLQ  $23, h
+	IMULQ prime2, h
+	ADDQ  ·primes+16(SB), h
 
-	MOVL  (SI), R8
-	ADDQ  $4, SI
-	IMULQ R13, R8
-	XORQ  R8, AX
-
-	ROLQ  $23, AX
-	IMULQ R14, AX
-	ADDQ  ·prime3v(SB), AX
-
-singles:
-	ADDQ $4, BX
-	CMPQ SI, BX
+try1:
+	ADDQ $4, end
+	CMPQ p, end
 	JGE  finalize
 
-singlesLoop:
-	MOVBQZX (SI), R12
-	ADDQ    $1, SI
-	IMULQ   ·prime5v(SB), R12
-	XORQ    R12, AX
+loop1:
+	MOVBQZX (p), x
+	ADDQ    $1, p
+	IMULQ   ·primes+32(SB), x
+	XORQ    x, h
+	ROLQ    $11, h
+	IMULQ   prime1, h
 
-	ROLQ  $11, AX
-	IMULQ R13, AX
-
-	CMPQ SI, BX
-	JL   singlesLoop
+	CMPQ p, end
+	JL   loop1
 
 finalize:
-	MOVQ  AX, R12
-	SHRQ  $33, R12
-	XORQ  R12, AX
-	IMULQ R14, AX
-	MOVQ  AX, R12
-	SHRQ  $29, R12
-	XORQ  R12, AX
-	IMULQ ·prime3v(SB), AX
-	MOVQ  AX, R12
-	SHRQ  $32, R12
-	XORQ  R12, AX
+	MOVQ  h, x
+	SHRQ  $33, x
+	XORQ  x, h
+	IMULQ prime2, h
+	MOVQ  h, x
+	SHRQ  $29, x
+	XORQ  x, h
+	IMULQ ·primes+16(SB), h
+	MOVQ  h, x
+	SHRQ  $32, x
+	XORQ  x, h
 
-	MOVQ AX, ret+24(FP)
+	MOVQ h, ret+24(FP)
 	RET
 
-// writeBlocks uses the same registers as above except that it uses AX to store
-// the d pointer.
-
 // func writeBlocks(d *Digest, b []byte) int
-TEXT ·writeBlocks(SB), NOSPLIT, $0-40
+TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
 	// Load fixed primes needed for round.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
+	MOVQ ·primes+0(SB), prime1
+	MOVQ ·primes+8(SB), prime2
 
 	// Load slice.
-	MOVQ b_base+8(FP), SI
-	MOVQ b_len+16(FP), DX
-	LEAQ (SI)(DX*1), BX
-	SUBQ $32, BX
+	MOVQ b_base+8(FP), p
+	MOVQ b_len+16(FP), n
+	LEAQ (p)(n*1), end
+	SUBQ $32, end
 
 	// Load vN from d.
-	MOVQ d+0(FP), AX
-	MOVQ 0(AX), R8   // v1
-	MOVQ 8(AX), R9   // v2
-	MOVQ 16(AX), R10 // v3
-	MOVQ 24(AX), R11 // v4
+	MOVQ s+0(FP), d
+	MOVQ 0(d), v1
+	MOVQ 8(d), v2
+	MOVQ 16(d), v3
+	MOVQ 24(d), v4
 
 	// We don't need to check the loop condition here; this function is
 	// always called with at least one block of data to process.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
-
-	CMPQ SI, BX
-	JLE  blockLoop
+	blockLoop()
 
 	// Copy vN back to d.
-	MOVQ R8, 0(AX)
-	MOVQ R9, 8(AX)
-	MOVQ R10, 16(AX)
-	MOVQ R11, 24(AX)
+	MOVQ v1, 0(d)
+	MOVQ v2, 8(d)
+	MOVQ v3, 16(d)
+	MOVQ v4, 24(d)
 
-	// The number of bytes written is SI minus the old base pointer.
-	SUBQ b_base+8(FP), SI
-	MOVQ SI, ret+32(FP)
+	// The number of bytes written is p minus the old base pointer.
+	SUBQ b_base+8(FP), p
+	MOVQ p, ret+32(FP)
 
 	RET

vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s

@@ -0,0 +1,183 @@
+//go:build !appengine && gc && !purego
+// +build !appengine
+// +build gc
+// +build !purego
+
+#include "textflag.h"
+
+// Registers:
+#define digest	R1
+#define h	R2 // return value
+#define p	R3 // input pointer
+#define n	R4 // input length
+#define nblocks	R5 // n / 32
+#define prime1	R7
+#define prime2	R8
+#define prime3	R9
+#define prime4	R10
+#define prime5	R11
+#define v1	R12
+#define v2	R13
+#define v3	R14
+#define v4	R15
+#define x1	R20
+#define x2	R21
+#define x3	R22
+#define x4	R23
+
+#define round(acc, x) \
+	MADD prime2, acc, x, acc \
+	ROR  $64-31, acc         \
+	MUL  prime1, acc
+
+// round0 performs the operation x = round(0, x).
+#define round0(x) \
+	MUL prime2, x \
+	ROR $64-31, x \
+	MUL prime1, x
+
+#define mergeRound(acc, x) \
+	round0(x)                     \
+	EOR  x, acc                   \
+	MADD acc, prime4, prime1, acc
+
+// blockLoop processes as many 32-byte blocks as possible,
+// updating v1, v2, v3, and v4. It assumes that n >= 32.
+#define blockLoop() \
+	LSR     $5, n, nblocks  \
+	PCALIGN $16             \
+	loop:                   \
+	LDP.P   16(p), (x1, x2) \
+	LDP.P   16(p), (x3, x4) \
+	round(v1, x1)           \
+	round(v2, x2)           \
+	round(v3, x3)           \
+	round(v4, x4)           \
+	SUB     $1, nblocks     \
+	CBNZ    nblocks, loop
+
+// func Sum64(b []byte) uint64
+TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32
+	LDP b_base+0(FP), (p, n)
+
+	LDP  ·primes+0(SB), (prime1, prime2)
+	LDP  ·primes+16(SB), (prime3, prime4)
+	MOVD ·primes+32(SB), prime5
+
+	CMP  $32, n
+	CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 }
+	BLT  afterLoop
+
+	ADD  prime1, prime2, v1
+	MOVD prime2, v2
+	MOVD $0, v3
+	NEG  prime1, v4
+
+	blockLoop()
+
+	ROR $64-1, v1, x1
+	ROR $64-7, v2, x2
+	ADD x1, x2
+	ROR $64-12, v3, x3
+	ROR $64-18, v4, x4
+	ADD x3, x4
+	ADD x2, x4, h
+
+	mergeRound(h, v1)
+	mergeRound(h, v2)
+	mergeRound(h, v3)
+	mergeRound(h, v4)
+
+afterLoop:
+	ADD n, h
+
+	TBZ   $4, n, try8
+	LDP.P 16(p), (x1, x2)
+
+	round0(x1)
+
+	// NOTE: here and below, sequencing the EOR after the ROR (using a
+	// rotated register) is worth a small but measurable speedup for small
+	// inputs.
+	ROR  $64-27, h
+	EOR  x1 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+	round0(x2)
+	ROR  $64-27, h
+	EOR  x2 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+try8:
+	TBZ    $3, n, try4
+	MOVD.P 8(p), x1
+
+	round0(x1)
+	ROR  $64-27, h
+	EOR  x1 @> 64-27, h, h
+	MADD h, prime4, prime1, h
+
+try4:
+	TBZ     $2, n, try2
+	MOVWU.P 4(p), x2
+
+	MUL  prime1, x2
+	ROR  $64-23, h
+	EOR  x2 @> 64-23, h, h
+	MADD h, prime3, prime2, h
+
+try2:
+	TBZ     $1, n, try1
+	MOVHU.P 2(p), x3
+	AND     $255, x3, x1
+	LSR     $8, x3, x2
+
+	MUL prime5, x1
+	ROR $64-11, h
+	EOR x1 @> 64-11, h, h
+	MUL prime1, h
+
+	MUL prime5, x2
+	ROR $64-11, h
+	EOR x2 @> 64-11, h, h
+	MUL prime1, h
+
+try1:
+	TBZ   $0, n, finalize
+	MOVBU (p), x4
+
+	MUL prime5, x4
+	ROR $64-11, h
+	EOR x4 @> 64-11, h, h
+	MUL prime1, h
+
+finalize:
+	EOR h >> 33, h
+	MUL prime2, h
+	EOR h >> 29, h
+	MUL prime3, h
+	EOR h >> 32, h
+
+	MOVD h, ret+24(FP)
+	RET
+
+// func writeBlocks(d *Digest, b []byte) int
+TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
+	LDP ·primes+0(SB), (prime1, prime2)
+
+	// Load state. Assume v[1-4] are stored contiguously.
+	MOVD d+0(FP), digest
+	LDP  0(digest), (v1, v2)
+	LDP  16(digest), (v3, v4)
+
+	LDP b_base+8(FP), (p, n)
+
+	blockLoop()
+
+	// Store updated state.
+	STP (v1, v2), 0(digest)
+	STP (v3, v4), 16(digest)
+
+	BIC  $31, n
+	MOVD n, ret+32(FP)
+	RET

vendor/github.com/cespare/xxhash/v2/xxhash_asm.go

@@ -1,3 +1,5 @@
+//go:build (amd64 || arm64) && !appengine && gc && !purego
+// +build amd64 arm64
 // +build !appengine
 // +build gc
 // +build !purego

vendor/github.com/cespare/xxhash/v2/xxhash_other.go

@@ -1,4 +1,5 @@
-// +build !amd64 appengine !gc purego
+//go:build (!amd64 && !arm64) || appengine || !gc || purego
+// +build !amd64,!arm64 appengine !gc purego
 
 package xxhash
 
@@ -14,10 +15,10 @@ func Sum64(b []byte) uint64 {
 	var h uint64
 
 	if n >= 32 {
-		v1 := prime1v + prime2
+		v1 := primes[0] + prime2
 		v2 := prime2
 		v3 := uint64(0)
-		v4 := -prime1v
+		v4 := -primes[0]
 		for len(b) >= 32 {
 			v1 = round(v1, u64(b[0:8:len(b)]))
 			v2 = round(v2, u64(b[8:16:len(b)]))
@@ -36,19 +37,18 @@ func Sum64(b []byte) uint64 {
 
 	h += uint64(n)
 
-	i, end := 0, len(b)
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(b[i:i+8:len(b)]))
+	for ; len(b) >= 8; b = b[8:] {
+		k1 := round(0, u64(b[:8]))
 		h ^= k1
 		h = rol27(h)*prime1 + prime4
 	}
-	if i+4 <= end {
-		h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
+	if len(b) >= 4 {
+		h ^= uint64(u32(b[:4])) * prime1
 		h = rol23(h)*prime2 + prime3
-		i += 4
+		b = b[4:]
 	}
-	for ; i < end; i++ {
-		h ^= uint64(b[i]) * prime5
+	for ; len(b) > 0; b = b[1:] {
+		h ^= uint64(b[0]) * prime5
 		h = rol11(h) * prime1
 	}
 

vendor/github.com/cespare/xxhash/v2/xxhash_safe.go

@@ -1,3 +1,4 @@
+//go:build appengine
 // +build appengine
 
 // This file contains the safe implementations of otherwise unsafe-using code.

vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go

@@ -1,3 +1,4 @@
+//go:build !appengine
 // +build !appengine
 
 // This file encapsulates usage of unsafe.
@@ -11,7 +12,7 @@ import (
 
 // In the future it's possible that compiler optimizations will make these
 // XxxString functions unnecessary by realizing that calls such as
-// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205.
+// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205.
 // If that happens, even if we keep these functions they can be replaced with
 // the trivial safe code.
 

vendor/github.com/dustin/go-humanize/.travis.yml

@@ -1,12 +1,12 @@
 sudo: false
 language: go
+go_import_path: github.com/dustin/go-humanize
 go:
-  - 1.3.x
-  - 1.5.x
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
-  - 1.9.x
+  - 1.13.x
+  - 1.14.x
+  - 1.15.x
+  - 1.16.x
+  - stable
   - master
 matrix:
   allow_failures:
@@ -15,7 +15,7 @@ matrix:
 install:
   - # Do nothing. This is needed to prevent default install action "go get -t -v ./..." from happening here (we want it to happen inside script step).
 script:
-  - go get -t -v ./...
   - diff -u <(echo -n) <(gofmt -d -s .)
-  - go tool vet .
+  - go vet .
+  - go install -v -race ./...
   - go test -v -race ./...

vendor/github.com/dustin/go-humanize/README.markdown

@@ -5,7 +5,7 @@ Just a few functions for helping humanize times and sizes.
 `go get` it as `github.com/dustin/go-humanize`, import it as
 `"github.com/dustin/go-humanize"`, use it as `humanize`.
 
-See [godoc](https://godoc.org/github.com/dustin/go-humanize) for
+See [godoc](https://pkg.go.dev/github.com/dustin/go-humanize) for
 complete documentation.
 
 ## Sizes

vendor/github.com/dustin/go-humanize/bigbytes.go

@@ -28,6 +28,10 @@ var (
 	BigZiByte = (&big.Int{}).Mul(BigEiByte, bigIECExp)
 	// BigYiByte is 1,024 z bytes in bit.Ints
 	BigYiByte = (&big.Int{}).Mul(BigZiByte, bigIECExp)
+	// BigRiByte is 1,024 y bytes in bit.Ints
+	BigRiByte = (&big.Int{}).Mul(BigYiByte, bigIECExp)
+	// BigQiByte is 1,024 r bytes in bit.Ints
+	BigQiByte = (&big.Int{}).Mul(BigRiByte, bigIECExp)
 )
 
 var (
@@ -51,6 +55,10 @@ var (
 	BigZByte = (&big.Int{}).Mul(BigEByte, bigSIExp)
 	// BigYByte is 1,000 SI z bytes in big.Ints
 	BigYByte = (&big.Int{}).Mul(BigZByte, bigSIExp)
+	// BigRByte is 1,000 SI y bytes in big.Ints
+	BigRByte = (&big.Int{}).Mul(BigYByte, bigSIExp)
+	// BigQByte is 1,000 SI r bytes in big.Ints
+	BigQByte = (&big.Int{}).Mul(BigRByte, bigSIExp)
 )
 
 var bigBytesSizeTable = map[string]*big.Int{
@@ -71,6 +79,10 @@ var bigBytesSizeTable = map[string]*big.Int{
 	"zb":  BigZByte,
 	"yib": BigYiByte,
 	"yb":  BigYByte,
+	"rib": BigRiByte,
+	"rb":  BigRByte,
+	"qib": BigQiByte,
+	"qb":  BigQByte,
 	// Without suffix
 	"":   BigByte,
 	"ki": BigKiByte,
@@ -89,6 +101,10 @@ var bigBytesSizeTable = map[string]*big.Int{
 	"zi": BigZiByte,
 	"y":  BigYByte,
 	"yi": BigYiByte,
+	"r":  BigRByte,
+	"ri": BigRiByte,
+	"q":  BigQByte,
+	"qi": BigQiByte,
 }
 
 var ten = big.NewInt(10)
@@ -115,7 +131,7 @@ func humanateBigBytes(s, base *big.Int, sizes []string) string {
 //
 // BigBytes(82854982) -> 83 MB
 func BigBytes(s *big.Int) string {
-	sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
+	sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", "RB", "QB"}
 	return humanateBigBytes(s, bigSIExp, sizes)
 }
 
@@ -125,7 +141,7 @@ func BigBytes(s *big.Int) string {
 //
 // BigIBytes(82854982) -> 79 MiB
 func BigIBytes(s *big.Int) string {
-	sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"}
+	sizes := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB", "RiB", "QiB"}
 	return humanateBigBytes(s, bigIECExp, sizes)
 }
 

vendor/github.com/dustin/go-humanize/commaf.go

@@ -1,3 +1,4 @@
+//go:build go1.6
 // +build go1.6
 
 package humanize

vendor/github.com/dustin/go-humanize/ftoa.go

@@ -6,6 +6,9 @@ import (
 )
 
 func stripTrailingZeros(s string) string {
+	if !strings.ContainsRune(s, '.') {
+		return s
+	}
 	offset := len(s) - 1
 	for offset > 0 {
 		if s[offset] == '.' {

vendor/github.com/dustin/go-humanize/number.go

@@ -73,7 +73,7 @@ func FormatFloat(format string, n float64) string {
 	if n > math.MaxFloat64 {
 		return "Infinity"
 	}
-	if n < -math.MaxFloat64 {
+	if n < (0.0 - math.MaxFloat64) {
 		return "-Infinity"
 	}
 

vendor/github.com/dustin/go-humanize/si.go

@@ -8,6 +8,8 @@ import (
 )
 
 var siPrefixTable = map[float64]string{
+	-30: "q", // quecto
+	-27: "r", // ronto
 	-24: "y", // yocto
 	-21: "z", // zepto
 	-18: "a", // atto
@@ -25,6 +27,8 @@ var siPrefixTable = map[float64]string{
 	18:  "E", // exa
 	21:  "Z", // zetta
 	24:  "Y", // yotta
+	27:  "R", // ronna
+	30:  "Q", // quetta
 }
 
 var revSIPrefixTable = revfmap(siPrefixTable)

vendor/github.com/golang-jwt/jwt/v4/README.md

@@ -54,9 +54,9 @@ import "github.com/golang-jwt/jwt/v4"
 
 See [the project documentation](https://pkg.go.dev/github.com/golang-jwt/jwt/v4) for examples of usage:
 
-* [Simple example of parsing and validating a token](https://pkg.go.dev/github.com/golang-jwt/jwt#example-Parse-Hmac)
-* [Simple example of building and signing a token](https://pkg.go.dev/github.com/golang-jwt/jwt#example-New-Hmac)
-* [Directory of Examples](https://pkg.go.dev/github.com/golang-jwt/jwt#pkg-examples)
+* [Simple example of parsing and validating a token](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#example-Parse-Hmac)
+* [Simple example of building and signing a token](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#example-New-Hmac)
+* [Directory of Examples](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#pkg-examples)
 
 ## Extensions
 
@@ -96,7 +96,7 @@ A token is simply a JSON object that is signed by its author. this tells you exa
 * The author of the token was in the possession of the signing secret
 * The data has not been modified since it was signed
 
-It's important to know that JWT does not provide encryption, which means anyone who has access to the token can read its contents. If you need to protect (encrypt) the data, there is a companion spec, `JWE`, that provides this functionality. JWE is currently outside the scope of this library.
+It's important to know that JWT does not provide encryption, which means anyone who has access to the token can read its contents. If you need to protect (encrypt) the data, there is a companion spec, `JWE`, that provides this functionality. The companion project https://github.com/golang-jwt/jwe aims at a (very) experimental implementation of the JWE standard.
 
 ### Choosing a Signing Method
 
@@ -110,10 +110,10 @@ Asymmetric signing methods, such as RSA, use different keys for signing and veri
 
 Each signing method expects a different object type for its signing keys. See the package documentation for details. Here are the most common ones:
 
-* The [HMAC signing method](https://pkg.go.dev/github.com/golang-jwt/jwt#SigningMethodHMAC) (`HS256`,`HS384`,`HS512`) expect `[]byte` values for signing and validation
-* The [RSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt#SigningMethodRSA) (`RS256`,`RS384`,`RS512`) expect `*rsa.PrivateKey` for signing and `*rsa.PublicKey` for validation
-* The [ECDSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt#SigningMethodECDSA) (`ES256`,`ES384`,`ES512`) expect `*ecdsa.PrivateKey` for signing and `*ecdsa.PublicKey` for validation
-* The [EdDSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt#SigningMethodEd25519) (`Ed25519`) expect `ed25519.PrivateKey` for signing and `ed25519.PublicKey` for validation
+* The [HMAC signing method](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#SigningMethodHMAC) (`HS256`,`HS384`,`HS512`) expect `[]byte` values for signing and validation
+* The [RSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#SigningMethodRSA) (`RS256`,`RS384`,`RS512`) expect `*rsa.PrivateKey` for signing and `*rsa.PublicKey` for validation
+* The [ECDSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#SigningMethodECDSA) (`ES256`,`ES384`,`ES512`) expect `*ecdsa.PrivateKey` for signing and `*ecdsa.PublicKey` for validation
+* The [EdDSA signing method](https://pkg.go.dev/github.com/golang-jwt/jwt/v4#SigningMethodEd25519) (`Ed25519`) expect `ed25519.PrivateKey` for signing and `ed25519.PublicKey` for validation
 
 ### JWT and OAuth
 
@@ -131,7 +131,7 @@ This library uses descriptive error messages whenever possible. If you are not g
 
 ## More
 
-Documentation can be found [on pkg.go.dev](https://pkg.go.dev/github.com/golang-jwt/jwt).
+Documentation can be found [on pkg.go.dev](https://pkg.go.dev/github.com/golang-jwt/jwt/v4).
 
 The command line utility included in this project (cmd/jwt) provides a straightforward example of token creation and parsing as well as a useful tool for debugging your own integration. You'll also find several implementation examples in the documentation.
 

vendor/github.com/golang-jwt/jwt/v4/claims.go

@@ -265,9 +265,5 @@ func verifyIss(iss string, cmp string, required bool) bool {
 	if iss == "" {
 		return !required
 	}
-	if subtle.ConstantTimeCompare([]byte(iss), []byte(cmp)) != 0 {
-		return true
-	} else {
-		return false
-	}
+	return subtle.ConstantTimeCompare([]byte(iss), []byte(cmp)) != 0
 }

vendor/github.com/golang-jwt/jwt/v4/parser.go

@@ -42,6 +42,13 @@ func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) {
 	return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc)
 }
 
+// ParseWithClaims parses, validates, and verifies like Parse, but supplies a default object implementing the Claims
+// interface. This provides default values which can be overridden and allows a caller to use their own type, rather
+// than the default MapClaims implementation of Claims.
+//
+// Note: If you provide a custom claim implementation that embeds one of the standard claims (such as RegisteredClaims),
+// make sure that a) you either embed a non-pointer version of the claims or b) if you are using a pointer, allocate the
+// proper memory for it before passing in the overall claims, otherwise you might run into a panic.
 func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) {
 	token, parts, err := p.ParseUnverified(tokenString, claims)
 	if err != nil {

vendor/github.com/golang-jwt/jwt/v4/token.go

@@ -99,6 +99,11 @@ func Parse(tokenString string, keyFunc Keyfunc, options ...ParserOption) (*Token
 	return NewParser(options...).Parse(tokenString, keyFunc)
 }
 
+// ParseWithClaims is a shortcut for NewParser().ParseWithClaims().
+//
+// Note: If you provide a custom claim implementation that embeds one of the standard claims (such as RegisteredClaims),
+// make sure that a) you either embed a non-pointer version of the claims or b) if you are using a pointer, allocate the
+// proper memory for it before passing in the overall claims, otherwise you might run into a panic.
 func ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc, options ...ParserOption) (*Token, error) {
 	return NewParser(options...).ParseWithClaims(tokenString, claims, keyFunc)
 }

vendor/github.com/klauspost/compress/s2/README.md

@@ -325,35 +325,35 @@ The content compressed in this mode is fully compatible with the standard decode
 
 Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU):
 
-| File                                                                                                | S2 speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
-|-----------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
-| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z)                    | 12.70x   | 10556 MB/s    | 7.35%        | 4.15x       | 3455 MB/s           | 12.79%             |
-| (1 CPU)                                                                                             | 1.14x    | 948 MB/s      | -            | 0.42x       | 349 MB/s            | -                  |
-| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 17.13x   | 14484 MB/s    | 31.60%       | 10.09x      | 8533 MB/s           | 37.71%             |
-| (1 CPU)                                                                                             | 1.33x    | 1127 MB/s     | -            | 0.70x       | 589 MB/s            | -                  |
-| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst)         | 15.14x   | 12000 MB/s    | -5.79%       | 6.59x       | 5223 MB/s           | 5.80%              |
-| (1 CPU)                                                                                             | 1.11x    | 877 MB/s      | -            | 0.47x       | 370 MB/s            | -                  |
-| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst)                     | 14.62x   | 12116 MB/s    | 15.90%       | 5.35x       | 4430 MB/s           | 16.08%             |
-| (1 CPU)                                                                                             | 1.38x    | 1146 MB/s     | -            | 0.38x       | 312 MB/s            | -                  |
-| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst)                             | 8.83x    | 17579 MB/s    | 43.86%       | 6.54x       | 13011 MB/s          | 47.23%             |
-| (1 CPU)                                                                                             | 1.14x    | 2259 MB/s     | -            | 0.74x       | 1475 MB/s           | -                  |
-| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z)                                    | 16.72x   | 14019 MB/s    | 24.02%       | 10.11x      | 8477 MB/s           | 30.48%             |
-| (1 CPU)                                                                                             | 1.24x    | 1043 MB/s     | -            | 0.70x       | 586 MB/s            | -                  |
-| [10gb.tar](http://mattmahoney.net/dc/10gb.html)                                                     | 13.33x   | 9254 MB/s     | 1.84%        | 6.75x       | 4686 MB/s           | 6.72%              |
-| (1 CPU)                                                                                             | 0.97x    | 672 MB/s      | -            | 0.53x       | 366 MB/s            | -                  |
-| sharnd.out.2gb                                                                                      | 2.11x    | 12639 MB/s    | 0.01%        | 1.98x       | 11833 MB/s          | 0.01%              |
-| (1 CPU)                                                                                             | 0.93x    | 5594 MB/s     | -            | 1.34x       | 8030 MB/s           | -                  |
-| [enwik9](http://mattmahoney.net/dc/textdata.html)                                                   | 19.34x   | 8220 MB/s     | 3.98%        | 7.87x       | 3345 MB/s           | 15.82%             |
-| (1 CPU)                                                                                             | 1.06x    | 452 MB/s      | -            | 0.50x       | 213 MB/s            | -                  |
-| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip)                                    | 10.48x   | 6124 MB/s     | 5.67%        | 3.76x       | 2197 MB/s           | 12.60%             |
-| (1 CPU)                                                                                             | 0.97x    | 568 MB/s      | -            | 0.46x       | 271 MB/s            | -                  |
-| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results)                                 | 21.07x   | 9020 MB/s     | 6.36%        | 6.91x       | 2959 MB/s           | 16.95%             |
-| (1 CPU)                                                                                             | 1.07x    | 460 MB/s      | -            | 0.51x       | 220 MB/s            | -                  |
+| File                                                                                                    | S2 Speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
+|---------------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
+| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z)                        | 16.33x   | 10556 MB/s    | 8.0%         | 6.04x       | 5252 MB/s           | 14.7%              |
+| (1 CPU)                                                                                                 | 1.08x    | 940 MB/s      | -            | 0.46x       | 400 MB/s            | -                  |
+| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst)     | 16.51x   | 15224 MB/s    | 31.70%       | 9.47x       | 8734 MB/s           | 37.71%             |
+| (1 CPU)                                                                                                 | 1.26x    | 1157 MB/s     | -            | 0.60x       | 556 MB/s            | -                  |
+| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst)             | 15.14x   | 12598 MB/s    | -5.76%       | 6.23x       | 5675 MB/s           | 3.62%              |
+| (1 CPU)                                                                                                 | 1.02x    | 932 MB/s      | -            | 0.47x       | 432 MB/s            | -                  |
+| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst)                         | 11.21x   | 12116 MB/s    | 15.95%       | 3.24x       | 3500 MB/s           | 18.00%             |
+| (1 CPU)                                                                                                 | 1.05x    | 1135 MB/s     | -            | 0.27x       | 292 MB/s            | -                  |
+| [apache.log](https://files.klauspost.com/compress/apache.log.zst)                                       | 8.55x    | 16673 MB/s    | 20.54%       | 5.85x       | 11420 MB/s          | 24.97%             |
+| (1 CPU)                                                                                                 | 1.91x    | 1771 MB/s     | -            | 0.53x       | 1041 MB/s           | -                  |
+| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z)                                        | 15.76x   | 14357 MB/s    | 24.01%       | 8.67x       | 7891 MB/s           | 33.68%             |
+| (1 CPU)                                                                                                 | 1.17x    | 1064 MB/s     | -            | 0.65x       | 595 MB/s            | -                  |
+| [10gb.tar](http://mattmahoney.net/dc/10gb.html)                                                         | 13.33x   | 9835 MB/s     | 2.34%        | 6.85x       | 4863 MB/s           | 9.96%              |
+| (1 CPU)                                                                                                 | 0.97x    | 689 MB/s      | -            | 0.55x       | 387 MB/s            | -                  |
+| sharnd.out.2gb                                                                                          | 9.11x    | 13213 MB/s    | 0.01%        | 1.49x       | 9184 MB/s           | 0.01%              |
+| (1 CPU)                                                                                                 | 0.88x    | 5418 MB/s     | -            | 0.77x       | 5417 MB/s           | -                  |
+| [sofia-air-quality-dataset csv](https://files.klauspost.com/compress/sofia-air-quality-dataset.tar.zst) | 22.00x   | 11477 MB/s    | 18.73%       | 11.15x      | 5817 MB/s           | 27.88%             |
+| (1 CPU)                                                                                                 | 1.23x    | 642 MB/s      | -            | 0.71x       | 642 MB/s            | -                  |
+| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip)                                        | 11.23x   | 6520 MB/s     | 5.9%         | 5.35x       | 3109 MB/s           | 15.88%             |
+| (1 CPU)                                                                                                 | 1.05x    | 607 MB/s      | -            | 0.52x       | 304 MB/s            | -                  |
+| [enwik9](https://files.klauspost.com/compress/enwik9.zst)                                               | 19.28x   | 8440 MB/s     | 4.04%        | 9.31x       | 4076 MB/s           | 18.04%             |
+| (1 CPU)                                                                                                 | 1.12x    | 488 MB/s      | -            | 0.57x       | 250 MB/s            | -                  |
 
 ### Legend
 
-* `S2 speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
-* `S2 throughput`: Throughput of S2 in MB/s. 
+* `S2 Speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
+* `S2 Throughput`: Throughput of S2 in MB/s. 
 * `S2 % smaller`: How many percent of the Snappy output size is S2 better.
 * `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy. 
 * `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy. 
@@ -361,7 +361,7 @@ Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all th
 
 There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads.
 
-Machine generated data gets by far the biggest compression boost, with size being being reduced by up to 45% of Snappy size.
+Machine generated data gets by far the biggest compression boost, with size being reduced by up to 35% of Snappy size.
 
 The "better" compression mode sees a good improvement in all cases, but usually at a performance cost.
 
@@ -404,15 +404,15 @@ The "better" compression mode will actively look for shorter matches, which is w
 Without assembly decompression is also very fast; single goroutine decompression speed. No assembly:
 
 | File                           | S2 Throughput | S2 throughput |
-|--------------------------------|--------------|---------------|
-| consensus.db.10gb.s2           | 1.84x        | 2289.8 MB/s   |
-| 10gb.tar.s2                    | 1.30x        | 867.07 MB/s   |
-| rawstudio-mint14.tar.s2        | 1.66x        | 1329.65 MB/s  |
-| github-june-2days-2019.json.s2 | 2.36x        | 1831.59 MB/s  |
-| github-ranks-backup.bin.s2     | 1.73x        | 1390.7 MB/s   |
-| enwik9.s2                      | 1.67x        | 681.53 MB/s   |
-| adresser.json.s2               | 3.41x        | 4230.53 MB/s  |
-| silesia.tar.s2                 | 1.52x        | 811.58        |
+|--------------------------------|---------------|---------------|
+| consensus.db.10gb.s2           | 1.84x         | 2289.8 MB/s   |
+| 10gb.tar.s2                    | 1.30x         | 867.07 MB/s   |
+| rawstudio-mint14.tar.s2        | 1.66x         | 1329.65 MB/s  |
+| github-june-2days-2019.json.s2 | 2.36x         | 1831.59 MB/s  |
+| github-ranks-backup.bin.s2     | 1.73x         | 1390.7 MB/s   |
+| enwik9.s2                      | 1.67x         | 681.53 MB/s   |
+| adresser.json.s2               | 3.41x         | 4230.53 MB/s  |
+| silesia.tar.s2                 | 1.52x         | 811.58        |
 
 Even though S2 typically compresses better than Snappy, decompression speed is always better. 
 
@@ -450,14 +450,14 @@ The most reliable is a wide dataset.
 For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
 53927 files, total input size: 4,014,735,833 bytes. Single goroutine used.
 
-| *                 | Input      | Output     | Reduction | MB/s   |
-|-------------------|------------|------------|-----------|--------|
-| S2                | 4014735833 | 1059723369 | 73.60%    | **934.34** |
-| S2 Better         | 4014735833 | 969670507  | 75.85%    | 532.70 |
-| S2 Best           | 4014735833 | 906625668  | **77.85%** | 46.84 |
-| Snappy            | 4014735833 | 1128706759 | 71.89%    | 762.59 |
-| S2, Snappy Output | 4014735833 | 1093821420 | 72.75%    | 908.60 |
-| LZ4               | 4014735833 | 1079259294 | 73.12%    | 526.94 |
+| *                 | Input      | Output     | Reduction  | MB/s       |
+|-------------------|------------|------------|------------|------------|
+| S2                | 4014735833 | 1059723369 | 73.60%     | **936.73** |
+| S2 Better         | 4014735833 | 961580539  | 76.05%     | 451.10     |
+| S2 Best           | 4014735833 | 899182886  | **77.60%** | 46.84      |
+| Snappy            | 4014735833 | 1128706759 | 71.89%     | 790.15     |
+| S2, Snappy Output | 4014735833 | 1093823291 | 72.75%     | 936.60     |
+| LZ4               | 4014735833 | 1063768713 | 73.50%     | 452.02     |
 
 S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best".
 "Better" mode provides the same compression speed as LZ4 with better compression ratio. 
@@ -489,43 +489,24 @@ AMD64 assembly is use for both S2 and Snappy.
 
 | Absolute Perf         | Snappy size | S2 Size | Snappy Speed | S2 Speed    | Snappy dec  | S2 dec      |
 |-----------------------|-------------|---------|--------------|-------------|-------------|-------------|
-| html                  | 22843       | 21111   | 16246 MB/s   | 17438 MB/s  | 40972 MB/s  | 49263 MB/s  |
-| urls.10K              | 335492      | 287326  | 7943 MB/s    | 9693 MB/s   | 22523 MB/s  | 26484 MB/s  |
-| fireworks.jpeg        | 123034      | 123100  | 349544 MB/s  | 273889 MB/s | 718321 MB/s | 827552 MB/s |
-| fireworks.jpeg (200B) | 146         | 155     | 8869 MB/s    | 17773 MB/s  | 33691 MB/s  | 52421 MB/s  |
-| paper-100k.pdf        | 85304       | 84459   | 167546 MB/s  | 101263 MB/s | 326905 MB/s | 291944 MB/s |
-| html_x_4              | 92234       | 21113   | 15194 MB/s   | 50670 MB/s  | 30843 MB/s  | 32217 MB/s  |
-| alice29.txt           | 88034       | 85975   | 5936 MB/s    | 6139 MB/s   | 12882 MB/s  | 20044 MB/s  |
-| asyoulik.txt          | 77503       | 79650   | 5517 MB/s    | 6366 MB/s   | 12735 MB/s  | 22806 MB/s  |
-| lcet10.txt            | 234661      | 220670  | 6235 MB/s    | 6067 MB/s   | 14519 MB/s  | 18697 MB/s  |
-| plrabn12.txt          | 319267      | 317985  | 5159 MB/s    | 5726 MB/s   | 11923 MB/s  | 19901 MB/s  |
-| geo.protodata         | 23335       | 18690   | 21220 MB/s   | 26529 MB/s  | 56271 MB/s  | 62540 MB/s  |
-| kppkn.gtb             | 69526       | 65312   | 9732 MB/s    | 8559 MB/s   | 18491 MB/s  | 18969 MB/s  |
-| alice29.txt (128B)    | 80          | 82      | 6691 MB/s    | 15489 MB/s  | 31883 MB/s  | 38874 MB/s  |
-| alice29.txt (1000B)   | 774         | 774     | 12204 MB/s   | 13000 MB/s  | 48056 MB/s  | 52341 MB/s  |
-| alice29.txt (10000B)  | 6648        | 6933    | 10044 MB/s   | 12806 MB/s  | 32378 MB/s  | 46322 MB/s  |
-| alice29.txt (20000B)  | 12686       | 13574   | 7733 MB/s    | 11210 MB/s  | 30566 MB/s  | 58969 MB/s  |
+| html                  | 22843       | 20868   | 16246 MB/s   | 18617 MB/s  | 40972 MB/s  | 49263 MB/s  |
+| urls.10K              | 335492      | 286541  | 7943 MB/s    | 10201 MB/s  | 22523 MB/s  | 26484 MB/s  |
+| fireworks.jpeg        | 123034      | 123100  | 349544 MB/s  | 303228 MB/s | 718321 MB/s | 827552 MB/s |
+| fireworks.jpeg (200B) | 146         | 155     | 8869 MB/s    | 20180 MB/s  | 33691 MB/s  | 52421 MB/s  |
+| paper-100k.pdf        | 85304       | 84202   | 167546 MB/s  | 112988 MB/s | 326905 MB/s | 291944 MB/s |
+| html_x_4              | 92234       | 20870   | 15194 MB/s   | 54457 MB/s  | 30843 MB/s  | 32217 MB/s  |
+| alice29.txt           | 88034       | 85934   | 5936 MB/s    | 6540 MB/s   | 12882 MB/s  | 20044 MB/s  |
+| asyoulik.txt          | 77503       | 79575   | 5517 MB/s    | 6657 MB/s   | 12735 MB/s  | 22806 MB/s  |
+| lcet10.txt            | 234661      | 220383  | 6235 MB/s    | 6303 MB/s   | 14519 MB/s  | 18697 MB/s  |
+| plrabn12.txt          | 319267      | 318196  | 5159 MB/s    | 6074 MB/s   | 11923 MB/s  | 19901 MB/s  |
+| geo.protodata         | 23335       | 18606   | 21220 MB/s   | 25432 MB/s  | 56271 MB/s  | 62540 MB/s  |
+| kppkn.gtb             | 69526       | 65019   | 9732 MB/s    | 8905 MB/s   | 18491 MB/s  | 18969 MB/s  |
+| alice29.txt (128B)    | 80          | 82      | 6691 MB/s    | 17179 MB/s  | 31883 MB/s  | 38874 MB/s  |
+| alice29.txt (1000B)   | 774         | 774     | 12204 MB/s   | 13273 MB/s  | 48056 MB/s  | 52341 MB/s  |
+| alice29.txt (10000B)  | 6648        | 6933    | 10044 MB/s   | 12824 MB/s  | 32378 MB/s  | 46322 MB/s  |
+| alice29.txt (20000B)  | 12686       | 13516   | 7733 MB/s    | 12160 MB/s  | 30566 MB/s  | 58969 MB/s  |
 
 
-| Relative Perf         | Snappy size | S2 size improved | S2 Speed | S2 Dec Speed |
-|-----------------------|-------------|------------------|----------|--------------|
-| html                  | 22.31%      | 7.58%            | 1.07x    | 1.20x        |
-| urls.10K              | 47.78%      | 14.36%           | 1.22x    | 1.18x        |
-| fireworks.jpeg        | 99.95%      | -0.05%           | 0.78x    | 1.15x        |
-| fireworks.jpeg (200B) | 73.00%      | -6.16%           | 2.00x    | 1.56x        |
-| paper-100k.pdf        | 83.30%      | 0.99%            | 0.60x    | 0.89x        |
-| html_x_4              | 22.52%      | 77.11%           | 3.33x    | 1.04x        |
-| alice29.txt           | 57.88%      | 2.34%            | 1.03x    | 1.56x        |
-| asyoulik.txt          | 61.91%      | -2.77%           | 1.15x    | 1.79x        |
-| lcet10.txt            | 54.99%      | 5.96%            | 0.97x    | 1.29x        |
-| plrabn12.txt          | 66.26%      | 0.40%            | 1.11x    | 1.67x        |
-| geo.protodata         | 19.68%      | 19.91%           | 1.25x    | 1.11x        |
-| kppkn.gtb             | 37.72%      | 6.06%            | 0.88x    | 1.03x        |
-| alice29.txt (128B)    | 62.50%      | -2.50%           | 2.31x    | 1.22x        |
-| alice29.txt (1000B)   | 77.40%      | 0.00%            | 1.07x    | 1.09x        |
-| alice29.txt (10000B)  | 66.48%      | -4.29%           | 1.27x    | 1.43x        |
-| alice29.txt (20000B)  | 63.43%      | -7.00%           | 1.45x    | 1.93x        |
-
 Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size. 
 
 Decompression speed is better than Snappy, except in one case. 
@@ -543,43 +524,24 @@ So individual benchmarks should only be seen as a guideline and the overall pict
 
 | Absolute Perf         | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec  | Better dec  |
 |-----------------------|-------------|-------------|--------------|--------------|-------------|-------------|
-| html                  | 22843       | 19833       | 16246 MB/s   | 7731 MB/s    | 40972 MB/s  | 40292 MB/s  |
-| urls.10K              | 335492      | 253529      | 7943 MB/s    | 3980 MB/s    | 22523 MB/s  | 20981 MB/s  |
-| fireworks.jpeg        | 123034      | 123100      | 349544 MB/s  | 9760 MB/s    | 718321 MB/s | 823698 MB/s |
-| fireworks.jpeg (200B) | 146         | 142         | 8869 MB/s    | 594 MB/s     | 33691 MB/s  | 30101 MB/s  |
-| paper-100k.pdf        | 85304       | 82915       | 167546 MB/s  | 7470 MB/s    | 326905 MB/s | 198869 MB/s |
-| html_x_4              | 92234       | 19841       | 15194 MB/s   | 23403 MB/s   | 30843 MB/s  | 30937 MB/s  |
-| alice29.txt           | 88034       | 73218       | 5936 MB/s    | 2945 MB/s    | 12882 MB/s  | 16611 MB/s  |
-| asyoulik.txt          | 77503       | 66844       | 5517 MB/s    | 2739 MB/s    | 12735 MB/s  | 14975 MB/s  |
-| lcet10.txt            | 234661      | 190589      | 6235 MB/s    | 3099 MB/s    | 14519 MB/s  | 16634 MB/s  |
-| plrabn12.txt          | 319267      | 270828      | 5159 MB/s    | 2600 MB/s    | 11923 MB/s  | 13382 MB/s  |
-| geo.protodata         | 23335       | 18278       | 21220 MB/s   | 11208 MB/s   | 56271 MB/s  | 57961 MB/s  |
-| kppkn.gtb             | 69526       | 61851       | 9732 MB/s    | 4556 MB/s    | 18491 MB/s  | 16524 MB/s  |
-| alice29.txt (128B)    | 80          | 81          | 6691 MB/s    | 529 MB/s     | 31883 MB/s  | 34225 MB/s  |
-| alice29.txt (1000B)   | 774         | 748         | 12204 MB/s   | 1943 MB/s    | 48056 MB/s  | 42068 MB/s  |
-| alice29.txt (10000B)  | 6648        | 6234        | 10044 MB/s   | 2949 MB/s    | 32378 MB/s  | 28813 MB/s  |
-| alice29.txt (20000B)  | 12686       | 11584       | 7733 MB/s    | 2822 MB/s    | 30566 MB/s  | 27315 MB/s  |
+| html                  | 22843       | 18972       | 16246 MB/s   | 8621 MB/s    | 40972 MB/s  | 40292 MB/s  |
+| urls.10K              | 335492      | 248079      | 7943 MB/s    | 5104 MB/s    | 22523 MB/s  | 20981 MB/s  |
+| fireworks.jpeg        | 123034      | 123100      | 349544 MB/s  | 84429 MB/s   | 718321 MB/s | 823698 MB/s |
+| fireworks.jpeg (200B) | 146         | 149         | 8869 MB/s    | 7125 MB/s    | 33691 MB/s  | 30101 MB/s  |
+| paper-100k.pdf        | 85304       | 82887       | 167546 MB/s  | 11087 MB/s   | 326905 MB/s | 198869 MB/s |
+| html_x_4              | 92234       | 18982       | 15194 MB/s   | 29316 MB/s   | 30843 MB/s  | 30937 MB/s  |
+| alice29.txt           | 88034       | 71611       | 5936 MB/s    | 3709 MB/s    | 12882 MB/s  | 16611 MB/s  |
+| asyoulik.txt          | 77503       | 65941       | 5517 MB/s    | 3380 MB/s    | 12735 MB/s  | 14975 MB/s  |
+| lcet10.txt            | 234661      | 184939      | 6235 MB/s    | 3537 MB/s    | 14519 MB/s  | 16634 MB/s  |
+| plrabn12.txt          | 319267      | 264990      | 5159 MB/s    | 2960 MB/s    | 11923 MB/s  | 13382 MB/s  |
+| geo.protodata         | 23335       | 17689       | 21220 MB/s   | 10859 MB/s   | 56271 MB/s  | 57961 MB/s  |
+| kppkn.gtb             | 69526       | 55398       | 9732 MB/s    | 5206 MB/s    | 18491 MB/s  | 16524 MB/s  |
+| alice29.txt (128B)    | 80          | 78          | 6691 MB/s    | 7422 MB/s    | 31883 MB/s  | 34225 MB/s  |
+| alice29.txt (1000B)   | 774         | 746         | 12204 MB/s   | 5734 MB/s    | 48056 MB/s  | 42068 MB/s  |
+| alice29.txt (10000B)  | 6648        | 6218        | 10044 MB/s   | 6055 MB/s    | 32378 MB/s  | 28813 MB/s  |
+| alice29.txt (20000B)  | 12686       | 11492       | 7733 MB/s    | 3143 MB/s    | 30566 MB/s  | 27315 MB/s  |
 
 
-| Relative Perf         | Snappy size | Better size | Better Speed | Better dec |
-|-----------------------|-------------|-------------|--------------|------------|
-| html                  | 22.31%      | 13.18%      | 0.48x        | 0.98x      |
-| urls.10K              | 47.78%      | 24.43%      | 0.50x        | 0.93x      |
-| fireworks.jpeg        | 99.95%      | -0.05%      | 0.03x        | 1.15x      |
-| fireworks.jpeg (200B) | 73.00%      | 2.74%       | 0.07x        | 0.89x      |
-| paper-100k.pdf        | 83.30%      | 2.80%       | 0.07x        | 0.61x      |
-| html_x_4              | 22.52%      | 78.49%      | 0.04x        | 1.00x      |
-| alice29.txt           | 57.88%      | 16.83%      | 1.54x        | 1.29x      |
-| asyoulik.txt          | 61.91%      | 13.75%      | 0.50x        | 1.18x      |
-| lcet10.txt            | 54.99%      | 18.78%      | 0.50x        | 1.15x      |
-| plrabn12.txt          | 66.26%      | 15.17%      | 0.50x        | 1.12x      |
-| geo.protodata         | 19.68%      | 21.67%      | 0.50x        | 1.03x      |
-| kppkn.gtb             | 37.72%      | 11.04%      | 0.53x        | 0.89x      |
-| alice29.txt (128B)    | 62.50%      | -1.25%      | 0.47x        | 1.07x      |
-| alice29.txt (1000B)   | 77.40%      | 3.36%       | 0.08x        | 0.88x      |
-| alice29.txt (10000B)  | 66.48%      | 6.23%       | 0.16x        | 0.89x      |
-| alice29.txt (20000B)  | 63.43%      | 8.69%       | 0.29x        | 0.89x      |
-
 Except for the mostly incompressible JPEG image compression is better and usually in the 
 double digits in terms of percentage reduction over Snappy.
 
@@ -605,29 +567,29 @@ Some examples compared on 16 core CPU, amd64 assembly used:
 
 ```
 * enwik10
-Default... 10000000000 -> 4761467548 [47.61%]; 1.098s, 8685.6MB/s
-Better...  10000000000 -> 4219438251 [42.19%]; 1.925s, 4954.2MB/s
-Best...    10000000000 -> 3627364337 [36.27%]; 43.051s, 221.5MB/s
+Default... 10000000000 -> 4759950115 [47.60%]; 1.03s, 9263.0MB/s
+Better...  10000000000 -> 4084706676 [40.85%]; 2.16s, 4415.4MB/s
+Best...    10000000000 -> 3615520079 [36.16%]; 42.259s, 225.7MB/s
 
 * github-june-2days-2019.json
-Default... 6273951764 -> 1043196283 [16.63%]; 431ms, 13882.3MB/s
-Better...  6273951764 -> 949146808 [15.13%]; 547ms, 10938.4MB/s
-Best...    6273951764 -> 832855506 [13.27%]; 9.455s, 632.8MB/s
+Default... 6273951764 -> 1041700255 [16.60%]; 431ms, 13882.3MB/s
+Better...  6273951764 -> 945841238 [15.08%]; 547ms, 10938.4MB/s
+Best...    6273951764 -> 826392576 [13.17%]; 9.455s, 632.8MB/s
 
 * nyc-taxi-data-10M.csv
-Default... 3325605752 -> 1095998837 [32.96%]; 324ms, 9788.7MB/s
-Better...  3325605752 -> 954776589 [28.71%]; 491ms, 6459.4MB/s
-Best...    3325605752 -> 779098746 [23.43%]; 8.29s, 382.6MB/s
+Default... 3325605752 -> 1093516949 [32.88%]; 324ms, 9788.7MB/s
+Better...  3325605752 -> 885394158 [26.62%]; 491ms, 6459.4MB/s
+Best...    3325605752 -> 773681257 [23.26%]; 8.29s, 412.0MB/s
 
 * 10gb.tar
-Default... 10065157632 -> 5916578242 [58.78%]; 1.028s, 9337.4MB/s
-Better...  10065157632 -> 5649207485 [56.13%]; 1.597s, 6010.6MB/s
-Best...    10065157632 -> 5208719802 [51.75%]; 32.78s, 292.8MB/
+Default... 10065157632 -> 5915541066 [58.77%]; 1.028s, 9337.4MB/s
+Better...  10065157632 -> 5453844650 [54.19%]; 1.597s, 4862.7MB/s
+Best...    10065157632 -> 5192495021 [51.59%]; 32.78s, 308.2MB/
 
 * consensus.db.10gb
-Default... 10737418240 -> 4562648848 [42.49%]; 882ms, 11610.0MB/s
-Better...  10737418240 -> 4542428129 [42.30%]; 1.533s, 6679.7MB/s
-Best...    10737418240 -> 4244773384 [39.53%]; 42.96s, 238.4MB/s
+Default... 10737418240 -> 4549762344 [42.37%]; 882ms, 12118.4MB/s
+Better...  10737418240 -> 4438535064 [41.34%]; 1.533s, 3500.9MB/s
+Best...    10737418240 -> 4210602774 [39.21%]; 42.96s, 254.4MB/s
 ```
 
 Decompression speed should be around the same as using the 'better' compression mode. 
@@ -648,10 +610,10 @@ If you would like more control, you can use the s2 package as described below:
 Snappy compatible blocks can be generated with the S2 encoder. 
 Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace 
 
-| Snappy                     | S2 replacement          |
-|----------------------------|-------------------------|
-| snappy.Encode(...)         | s2.EncodeSnappy(...)   |
-| snappy.MaxEncodedLen(...)  | s2.MaxEncodedLen(...)   |
+| Snappy                    | S2 replacement        |
+|---------------------------|-----------------------|
+| snappy.Encode(...)        | s2.EncodeSnappy(...)  |
+| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
 
 `s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output. 
 
@@ -660,12 +622,12 @@ Compression and speed is typically a bit better `MaxEncodedLen` is also smaller
 Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
 53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
 
-| Encoder               | Size       | MB/s       | Reduction |
-|-----------------------|------------|------------|------------
-| snappy.Encode         | 1128706759 | 725.59     | 71.89%    |
-| s2.EncodeSnappy       | 1093823291 | **899.16** | 72.75%    |
-| s2.EncodeSnappyBetter | 1001158548 | 578.49     | 75.06%    |
-| s2.EncodeSnappyBest   | 944507998  | 66.00      | **76.47%**|
+| Encoder               | Size       | MB/s       | Reduction  |
+|-----------------------|------------|------------|------------|
+| snappy.Encode         | 1128706759 | 725.59     | 71.89%     |
+| s2.EncodeSnappy       | 1093823291 | **899.16** | 72.75%     |
+| s2.EncodeSnappyBetter | 1001158548 | 578.49     | 75.06%     |
+| s2.EncodeSnappyBest   | 944507998  | 66.00      | **76.47%** |
 
 ## Streams
 
@@ -835,6 +797,13 @@ This is done using the regular "Skip" function:
 
 This will ensure that we are at exactly the offset we want, and reading from `dec` will start at the requested offset.
 
+# Compact storage
+
+For compact storage [RemoveIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RemoveIndexHeaders) can be used to remove any redundant info from 
+a serialized index. If you remove the header it must be restored before [Loading](https://pkg.go.dev/github.com/klauspost/compress/s2#Index.Load).
+
+This is expected to save 20 bytes. These can be restored using [RestoreIndexHeaders](https://pkg.go.dev/github.com/klauspost/compress/s2#RestoreIndexHeaders). This removes a layer of security, but is the most compact representation. Returns nil if headers contains errors.
+
 ## Index Format:
 
 Each block is structured as a snappy skippable block, with the chunk ID 0x99.
@@ -844,20 +813,20 @@ The block can be read from the front, but contains information so it can be read
 Numbers are stored as fixed size little endian values or [zigzag encoded](https://developers.google.com/protocol-buffers/docs/encoding#signed_integers) [base 128 varints](https://developers.google.com/protocol-buffers/docs/encoding), 
 with un-encoded value length of 64 bits, unless other limits are specified. 
 
-| Content                                                                   | Format                                                                                                                      |
-|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
-| ID, `[1]byte`                                                           | Always 0x99.                                                                                                                  |
-| Data Length, `[3]byte`                                                  | 3 byte little-endian length of the chunk in bytes, following this.                                                            |
-| Header `[6]byte`                                                        | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00".                                                        |
-| UncompressedSize, Varint                                                | Total Uncompressed size.                                                                                                      |
-| CompressedSize, Varint                                                  | Total Compressed size if known. Should be -1 if unknown.                                                                      |
-| EstBlockSize, Varint                                                    | Block Size, used for guessing uncompressed offsets. Must be >= 0.                                                             |
-| Entries, Varint                                                         | Number of Entries in index, must be < 65536 and >=0.                                                                          |
-| HasUncompressedOffsets `byte`                                           | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid.                                             |
-| UncompressedOffsets, [Entries]VarInt                                    | Uncompressed offsets. See below how to decode.                                                                                |
-| CompressedOffsets, [Entries]VarInt                                      | Compressed offsets. See below how to decode.                                                                                  |
-| Block Size, `[4]byte`                                                   | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block.       |
-| Trailer `[6]byte`                                                       | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. |
+| Content                              | Format                                                                                                                        |
+|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| ID, `[1]byte`                        | Always 0x99.                                                                                                                  |
+| Data Length, `[3]byte`               | 3 byte little-endian length of the chunk in bytes, following this.                                                            |
+| Header `[6]byte`                     | Header, must be `[115, 50, 105, 100, 120, 0]` or in text: "s2idx\x00".                                                        |
+| UncompressedSize, Varint             | Total Uncompressed size.                                                                                                      |
+| CompressedSize, Varint               | Total Compressed size if known. Should be -1 if unknown.                                                                      |
+| EstBlockSize, Varint                 | Block Size, used for guessing uncompressed offsets. Must be >= 0.                                                             |
+| Entries, Varint                      | Number of Entries in index, must be < 65536 and >=0.                                                                          |
+| HasUncompressedOffsets `byte`        | 0 if no uncompressed offsets are present, 1 if present. Other values are invalid.                                             |
+| UncompressedOffsets, [Entries]VarInt | Uncompressed offsets. See below how to decode.                                                                                |
+| CompressedOffsets, [Entries]VarInt   | Compressed offsets. See below how to decode.                                                                                  |
+| Block Size, `[4]byte`                | Little Endian total encoded size (including header and trailer). Can be used for searching backwards to start of block.       |
+| Trailer `[6]byte`                    | Trailer, must be `[0, 120, 100, 105, 50, 115]` or in text: "\x00xdi2s". Can be used for identifying block from end of stream. |
 
 For regular streams the uncompressed offsets are fully predictable,
 so `HasUncompressedOffsets` allows to specify that compressed blocks all have 
@@ -929,6 +898,7 @@ To decode from any given uncompressed offset `(wantOffset)`:
 
 See [using indexes](https://github.com/klauspost/compress/tree/master/s2#using-indexes) for functions that perform the operations with a simpler interface.
 
+
 # Format Extensions
 
 * Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
@@ -951,10 +921,11 @@ The length is specified by reading the 3-bit length specified in the tag and dec
 | 7      | 65540 + read 3 bytes |
 
 This allows any repeat offset + length to be represented by 2 to 5 bytes.
+It also allows to emit matches longer than 64 bytes with one copy + one repeat instead of several 64 byte copies.
 
 Lengths are stored as little endian values.
 
-The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams.
+The first copy of a block cannot be a repeat offset and the offset is reset on every block in streams.
 
 Default streaming block size is 1MB.
 

vendor/github.com/klauspost/compress/s2/decode.go

@@ -952,7 +952,11 @@ func (r *Reader) ReadSeeker(random bool, index []byte) (*ReadSeeker, error) {
 // Seek allows seeking in compressed data.
 func (r *ReadSeeker) Seek(offset int64, whence int) (int64, error) {
 	if r.err != nil {
-		return 0, r.err
+		if !errors.Is(r.err, io.EOF) {
+			return 0, r.err
+		}
+		// Reset on EOF
+		r.err = nil
 	}
 	if offset == 0 && whence == io.SeekCurrent {
 		return r.blockStart + int64(r.i), nil

vendor/github.com/klauspost/compress/s2/decode_other.go

@@ -28,6 +28,9 @@ func s2Decode(dst, src []byte) int {
 
 	// As long as we can read at least 5 bytes...
 	for s < len(src)-5 {
+		// Removing bounds checks is SLOWER, when if doing
+		// in := src[s:s+5]
+		// Checked on Go 1.18
 		switch src[s] & 0x03 {
 		case tagLiteral:
 			x := uint32(src[s] >> 2)
@@ -38,14 +41,19 @@ func s2Decode(dst, src []byte) int {
 				s += 2
 				x = uint32(src[s-1])
 			case x == 61:
+				in := src[s : s+3]
+				x = uint32(in[1]) | uint32(in[2])<<8
 				s += 3
-				x = uint32(src[s-2]) | uint32(src[s-1])<<8
 			case x == 62:
+				in := src[s : s+4]
+				// Load as 32 bit and shift down.
+				x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
+				x >>= 8
 				s += 4
-				x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
 			case x == 63:
+				in := src[s : s+5]
+				x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
 				s += 5
-				x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
 			}
 			length = int(x) + 1
 			if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
@@ -62,8 +70,8 @@ func s2Decode(dst, src []byte) int {
 
 		case tagCopy1:
 			s += 2
-			length = int(src[s-2]) >> 2 & 0x7
 			toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
+			length = int(src[s-2]) >> 2 & 0x7
 			if toffset == 0 {
 				if debug {
 					fmt.Print("(repeat) ")
@@ -71,14 +79,16 @@ func s2Decode(dst, src []byte) int {
 				// keep last offset
 				switch length {
 				case 5:
+					length = int(src[s]) + 4
 					s += 1
-					length = int(uint32(src[s-1])) + 4
 				case 6:
+					in := src[s : s+2]
+					length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
 					s += 2
-					length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
 				case 7:
+					in := src[s : s+3]
+					length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
 					s += 3
-					length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
 				default: // 0-> 4
 				}
 			} else {
@@ -86,14 +96,16 @@ func s2Decode(dst, src []byte) int {
 			}
 			length += 4
 		case tagCopy2:
+			in := src[s : s+3]
+			offset = int(uint32(in[1]) | uint32(in[2])<<8)
+			length = 1 + int(in[0])>>2
 			s += 3
-			length = 1 + int(src[s-3])>>2
-			offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
 
 		case tagCopy4:
+			in := src[s : s+5]
+			offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
+			length = 1 + int(in[0])>>2
 			s += 5
-			length = 1 + int(src[s-5])>>2
-			offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
 		}
 
 		if offset <= 0 || d < offset || length > len(dst)-d {

vendor/github.com/klauspost/compress/s2/encode_all.go

@@ -58,8 +58,9 @@ func encodeGo(dst, src []byte) []byte {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockGo(dst, src []byte) (d int) {
 	// Initialize the hash table.
 	const (

vendor/github.com/klauspost/compress/s2/encode_amd64.go

@@ -8,8 +8,9 @@ package s2
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlock(dst, src []byte) (d int) {
 	const (
 		// Use 12 bit table when less than...
@@ -43,8 +44,9 @@ func encodeBlock(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBetter(dst, src []byte) (d int) {
 	const (
 		// Use 12 bit table when less than...
@@ -78,8 +80,9 @@ func encodeBlockBetter(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockSnappy(dst, src []byte) (d int) {
 	const (
 		// Use 12 bit table when less than...
@@ -112,8 +115,9 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBetterSnappy(dst, src []byte) (d int) {
 	const (
 		// Use 12 bit table when less than...

vendor/github.com/klauspost/compress/s2/encode_best.go

@@ -15,8 +15,9 @@ import (
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBest(dst, src []byte) (d int) {
 	// Initialize the hash tables.
 	const (
@@ -176,14 +177,21 @@ func encodeBlockBest(dst, src []byte) (d int) {
 						best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
 					}
 					// Search for a match at best match end, see if that is better.
-					if sAt := best.s + best.length; sAt < sLimit {
-						sBack := best.s
-						backL := best.length
+					// Allow some bytes at the beginning to mismatch.
+					// Sweet spot is around 1-2 bytes, but depends on input.
+					// The skipped bytes are tested in Extend backwards,
+					// and still picked up as part of the match if they do.
+					const skipBeginning = 2
+					const skipEnd = 1
+					if sAt := best.s + best.length - skipEnd; sAt < sLimit {
+
+						sBack := best.s + skipBeginning - skipEnd
+						backL := best.length - skipBeginning
 						// Load initial values
 						cv = load64(src, sBack)
-						// Search for mismatch
+
+						// Grab candidates...
 						next := lTable[hash8(load64(src, sAt), lTableBits)]
-						//next := sTable[hash4(load64(src, sAt), sTableBits)]
 
 						if checkAt := getCur(next) - backL; checkAt > 0 {
 							best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
@@ -191,6 +199,16 @@ func encodeBlockBest(dst, src []byte) (d int) {
 						if checkAt := getPrev(next) - backL; checkAt > 0 {
 							best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
 						}
+						// Disabled: Extremely small gain
+						if false {
+							next = sTable[hash4(load64(src, sAt), sTableBits)]
+							if checkAt := getCur(next) - backL; checkAt > 0 {
+								best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
+							}
+							if checkAt := getPrev(next) - backL; checkAt > 0 {
+								best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
+							}
+						}
 					}
 				}
 			}
@@ -288,8 +306,9 @@ emitRemainder:
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBestSnappy(dst, src []byte) (d int) {
 	// Initialize the hash tables.
 	const (
@@ -546,6 +565,7 @@ emitRemainder:
 // emitCopySize returns the size to encode the offset+length
 //
 // It assumes that:
+//
 //	1 <= offset && offset <= math.MaxUint32
 //	4 <= length && length <= 1 << 24
 func emitCopySize(offset, length int) int {
@@ -584,6 +604,7 @@ func emitCopySize(offset, length int) int {
 // emitCopyNoRepeatSize returns the size to encode the offset+length
 //
 // It assumes that:
+//
 //	1 <= offset && offset <= math.MaxUint32
 //	4 <= length && length <= 1 << 24
 func emitCopyNoRepeatSize(offset, length int) int {

vendor/github.com/klauspost/compress/s2/encode_better.go

@@ -42,8 +42,9 @@ func hash8(u uint64, h uint8) uint32 {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBetterGo(dst, src []byte) (d int) {
 	// sLimit is when to stop looking for offset/length copies. The inputMargin
 	// lets us use a fast path for emitLiteral in the main loop, while we are
@@ -56,7 +57,7 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
 	// Initialize the hash tables.
 	const (
 		// Long hash matches.
-		lTableBits    = 16
+		lTableBits    = 17
 		maxLTableSize = 1 << lTableBits
 
 		// Short hash matches.
@@ -97,9 +98,26 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
 			lTable[hashL] = uint32(s)
 			sTable[hashS] = uint32(s)
 
+			valLong := load64(src, candidateL)
+			valShort := load64(src, candidateS)
+
+			// If long matches at least 8 bytes, use that.
+			if cv == valLong {
+				break
+			}
+			if cv == valShort {
+				candidateL = candidateS
+				break
+			}
+
 			// Check repeat at offset checkRep.
 			const checkRep = 1
-			if false && uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
+			// Minimum length of a repeat. Tested with various values.
+			// While 4-5 offers improvements in some, 6 reduces
+			// regressions significantly.
+			const wantRepeatBytes = 6
+			const repeatMask = ((1 << (wantRepeatBytes * 8)) - 1) << (8 * checkRep)
+			if false && repeat > 0 && cv&repeatMask == load64(src, s-repeat)&repeatMask {
 				base := s + checkRep
 				// Extend back
 				for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
@@ -109,8 +127,8 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
 				d += emitLiteral(dst[d:], src[nextEmit:base])
 
 				// Extend forward
-				candidate := s - repeat + 4 + checkRep
-				s += 4 + checkRep
+				candidate := s - repeat + wantRepeatBytes + checkRep
+				s += wantRepeatBytes + checkRep
 				for s < len(src) {
 					if len(src)-s < 8 {
 						if src[s] == src[candidate] {
@@ -127,28 +145,40 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
 					s += 8
 					candidate += 8
 				}
-				if nextEmit > 0 {
-					// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
-					d += emitRepeat(dst[d:], repeat, s-base)
-				} else {
-					// First match, cannot be repeat.
-					d += emitCopy(dst[d:], repeat, s-base)
-				}
+				// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
+				d += emitRepeat(dst[d:], repeat, s-base)
 				nextEmit = s
 				if s >= sLimit {
 					goto emitRemainder
 				}
+				// Index in-between
+				index0 := base + 1
+				index1 := s - 2
+
+				cv = load64(src, s)
+				for index0 < index1 {
+					cv0 := load64(src, index0)
+					cv1 := load64(src, index1)
+					lTable[hash7(cv0, lTableBits)] = uint32(index0)
+					sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
+
+					lTable[hash7(cv1, lTableBits)] = uint32(index1)
+					sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
+					index0 += 2
+					index1 -= 2
+				}
 
 				cv = load64(src, s)
 				continue
 			}
 
-			if uint32(cv) == load32(src, candidateL) {
+			// Long likely matches 7, so take that.
+			if uint32(cv) == uint32(valLong) {
 				break
 			}
 
 			// Check our short candidate
-			if uint32(cv) == load32(src, candidateS) {
+			if uint32(cv) == uint32(valShort) {
 				// Try a long candidate at s+1
 				hashL = hash7(cv>>8, lTableBits)
 				candidateL = int(lTable[hashL])
@@ -227,21 +257,29 @@ func encodeBlockBetterGo(dst, src []byte) (d int) {
 			// Do we have space for more, if not bail.
 			return 0
 		}
-		// Index match start+1 (long) and start+2 (short)
+
+		// Index short & long
 		index0 := base + 1
-		// Index match end-2 (long) and end-1 (short)
 		index1 := s - 2
 
 		cv0 := load64(src, index0)
 		cv1 := load64(src, index1)
-		cv = load64(src, s)
 		lTable[hash7(cv0, lTableBits)] = uint32(index0)
-		lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
-		lTable[hash7(cv1, lTableBits)] = uint32(index1)
-		lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
 		sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
-		sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
+
+		lTable[hash7(cv1, lTableBits)] = uint32(index1)
 		sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
+		index0 += 1
+		index1 -= 1
+		cv = load64(src, s)
+
+		// index every second long in between.
+		for index0 < index1 {
+			lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
+			lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
+			index0 += 2
+			index1 -= 2
+		}
 	}
 
 emitRemainder:
@@ -260,8 +298,9 @@ emitRemainder:
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src)) &&
-// 	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
+//	minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
 func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
 	// sLimit is when to stop looking for offset/length copies. The inputMargin
 	// lets us use a fast path for emitLiteral in the main loop, while we are
@@ -402,21 +441,29 @@ func encodeBlockBetterSnappyGo(dst, src []byte) (d int) {
 			// Do we have space for more, if not bail.
 			return 0
 		}
-		// Index match start+1 (long) and start+2 (short)
+
+		// Index short & long
 		index0 := base + 1
-		// Index match end-2 (long) and end-1 (short)
 		index1 := s - 2
 
 		cv0 := load64(src, index0)
 		cv1 := load64(src, index1)
-		cv = load64(src, s)
 		lTable[hash7(cv0, lTableBits)] = uint32(index0)
-		lTable[hash7(cv0>>8, lTableBits)] = uint32(index0 + 1)
-		lTable[hash7(cv1, lTableBits)] = uint32(index1)
-		lTable[hash7(cv1>>8, lTableBits)] = uint32(index1 + 1)
 		sTable[hash4(cv0>>8, sTableBits)] = uint32(index0 + 1)
-		sTable[hash4(cv0>>16, sTableBits)] = uint32(index0 + 2)
+
+		lTable[hash7(cv1, lTableBits)] = uint32(index1)
 		sTable[hash4(cv1>>8, sTableBits)] = uint32(index1 + 1)
+		index0 += 1
+		index1 -= 1
+		cv = load64(src, s)
+
+		// index every second long in between.
+		for index0 < index1 {
+			lTable[hash7(load64(src, index0), lTableBits)] = uint32(index0)
+			lTable[hash7(load64(src, index1), lTableBits)] = uint32(index1)
+			index0 += 2
+			index1 -= 2
+		}
 	}
 
 emitRemainder:

vendor/github.com/klauspost/compress/s2/encode_go.go

@@ -12,6 +12,7 @@ import (
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src))
 func encodeBlock(dst, src []byte) (d int) {
 	if len(src) < minNonLiteralBlockSize {
@@ -25,6 +26,7 @@ func encodeBlock(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src))
 func encodeBlockBetter(dst, src []byte) (d int) {
 	return encodeBlockBetterGo(dst, src)
@@ -35,6 +37,7 @@ func encodeBlockBetter(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src))
 func encodeBlockBetterSnappy(dst, src []byte) (d int) {
 	return encodeBlockBetterSnappyGo(dst, src)
@@ -45,6 +48,7 @@ func encodeBlockBetterSnappy(dst, src []byte) (d int) {
 // been written.
 //
 // It also assumes that:
+//
 //	len(dst) >= MaxEncodedLen(len(src))
 func encodeBlockSnappy(dst, src []byte) (d int) {
 	if len(src) < minNonLiteralBlockSize {
@@ -56,6 +60,7 @@ func encodeBlockSnappy(dst, src []byte) (d int) {
 // emitLiteral writes a literal chunk and returns the number of bytes written.
 //
 // It assumes that:
+//
 //	dst is long enough to hold the encoded bytes
 //	0 <= len(lit) && len(lit) <= math.MaxUint32
 func emitLiteral(dst, lit []byte) int {
@@ -146,6 +151,7 @@ func emitRepeat(dst []byte, offset, length int) int {
 // emitCopy writes a copy chunk and returns the number of bytes written.
 //
 // It assumes that:
+//
 //	dst is long enough to hold the encoded bytes
 //	1 <= offset && offset <= math.MaxUint32
 //	4 <= length && length <= 1 << 24
@@ -214,6 +220,7 @@ func emitCopy(dst []byte, offset, length int) int {
 // emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
 //
 // It assumes that:
+//
 //	dst is long enough to hold the encoded bytes
 //	1 <= offset && offset <= math.MaxUint32
 //	4 <= length && length <= 1 << 24
@@ -273,8 +280,8 @@ func emitCopyNoRepeat(dst []byte, offset, length int) int {
 // matchLen returns how many bytes match in a and b
 //
 // It assumes that:
-//   len(a) <= len(b)
 //
+//	len(a) <= len(b)
 func matchLen(a []byte, b []byte) int {
 	b = b[:len(a)]
 	var checked int

vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go

@@ -1,7 +1,6 @@
 // Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.
 
 //go:build !appengine && !noasm && gc && !noasm
-// +build !appengine,!noasm,gc,!noasm
 
 package s2
 
@@ -150,8 +149,9 @@ func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte) int
 // emitLiteral writes a literal chunk and returns the number of bytes written.
 //
 // It assumes that:
-//   dst is long enough to hold the encoded bytes with margin of 0 bytes
-//   0 <= len(lit) && len(lit) <= math.MaxUint32
+//
+//	dst is long enough to hold the encoded bytes with margin of 0 bytes
+//	0 <= len(lit) && len(lit) <= math.MaxUint32
 //
 //go:noescape
 func emitLiteral(dst []byte, lit []byte) int
@@ -165,9 +165,10 @@ func emitRepeat(dst []byte, offset int, length int) int
 // emitCopy writes a copy chunk and returns the number of bytes written.
 //
 // It assumes that:
-//   dst is long enough to hold the encoded bytes
-//   1 <= offset && offset <= math.MaxUint32
-//   4 <= length && length <= 1 << 24
+//
+//	dst is long enough to hold the encoded bytes
+//	1 <= offset && offset <= math.MaxUint32
+//	4 <= length && length <= 1 << 24
 //
 //go:noescape
 func emitCopy(dst []byte, offset int, length int) int
@@ -175,9 +176,10 @@ func emitCopy(dst []byte, offset int, length int) int
 // emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
 //
 // It assumes that:
-//   dst is long enough to hold the encoded bytes
-//   1 <= offset && offset <= math.MaxUint32
-//   4 <= length && length <= 1 << 24
+//
+//	dst is long enough to hold the encoded bytes
+//	1 <= offset && offset <= math.MaxUint32
+//	4 <= length && length <= 1 << 24
 //
 //go:noescape
 func emitCopyNoRepeat(dst []byte, offset int, length int) int
@@ -185,7 +187,8 @@ func emitCopyNoRepeat(dst []byte, offset int, length int) int
 // matchLen returns how many bytes match in a and b
 //
 // It assumes that:
-//   len(a) <= len(b)
+//
+//	len(a) <= len(b)
 //
 //go:noescape
 func matchLen(a []byte, b []byte) int

vendor/github.com/klauspost/cpuid/v2/README.md

@@ -16,10 +16,17 @@ Package home: https://github.com/klauspost/cpuid
 
 ## installing
 
-`go get -u github.com/klauspost/cpuid/v2` using modules. 
-
+`go get -u github.com/klauspost/cpuid/v2` using modules.
 Drop `v2` for others.
 
+### Homebrew
+
+For macOS/Linux users, you can install via [brew](https://brew.sh/)
+
+```sh
+$ brew install cpuid
+```
+
 ## example
 
 ```Go
@@ -77,10 +84,14 @@ We have Streaming SIMD 2 Extensions
 The `cpuid.CPU` provides access to CPU features. Use `cpuid.CPU.Supports()` to check for CPU features.
 A faster `cpuid.CPU.Has()` is provided which will usually be inlined by the gc compiler.  
 
+To test a larger number of features, they can be combined using `f := CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2)`, etc.
+This can be using with `cpuid.CPU.HasAll(f)` to quickly test if all features are supported.
+
 Note that for some cpu/os combinations some features will not be detected.
 `amd64` has rather good support and should work reliably on all platforms.
 
-Note that hypervisors may not pass through all CPU features.
+Note that hypervisors may not pass through all CPU features through to the guest OS,
+so even if your host supports a feature it may not be visible on guests.
 
 ## arm64 feature detection
 
@@ -253,6 +264,218 @@ Exit Code 0
 Exit Code 1
 ```
 
+
+## Available flags
+
+### x86 & amd64 
+
+| Feature Flag       | Description                                                                                                                                                                        |
+|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ADX                | Intel ADX (Multi-Precision Add-Carry Instruction Extensions)                                                                                                                       |
+| AESNI              | Advanced Encryption Standard New Instructions                                                                                                                                      |
+| AMD3DNOW           | AMD 3DNOW                                                                                                                                                                          |
+| AMD3DNOWEXT        | AMD 3DNowExt                                                                                                                                                                       |
+| AMXBF16            | Tile computational operations on BFLOAT16 numbers                                                                                                                                  |
+| AMXINT8            | Tile computational operations on 8-bit integers                                                                                                                                    |
+| AMXFP16            | Tile computational operations on FP16 numbers                                                                                                                                      |
+| AMXTILE            | Tile architecture                                                                                                                                                                  |
+| AVX                | AVX functions                                                                                                                                                                      |
+| AVX2               | AVX2 functions                                                                                                                                                                     |
+| AVX512BF16         | AVX-512 BFLOAT16 Instructions                                                                                                                                                      |
+| AVX512BITALG       | AVX-512 Bit Algorithms                                                                                                                                                             |
+| AVX512BW           | AVX-512 Byte and Word Instructions                                                                                                                                                 |
+| AVX512CD           | AVX-512 Conflict Detection Instructions                                                                                                                                            |
+| AVX512DQ           | AVX-512 Doubleword and Quadword Instructions                                                                                                                                       |
+| AVX512ER           | AVX-512 Exponential and Reciprocal Instructions                                                                                                                                    |
+| AVX512F            | AVX-512 Foundation                                                                                                                                                                 |
+| AVX512FP16         | AVX-512 FP16 Instructions                                                                                                                                                          |
+| AVX512IFMA         | AVX-512 Integer Fused Multiply-Add Instructions                                                                                                                                    |
+| AVX512PF           | AVX-512 Prefetch Instructions                                                                                                                                                      |
+| AVX512VBMI         | AVX-512 Vector Bit Manipulation Instructions                                                                                                                                       |
+| AVX512VBMI2        | AVX-512 Vector Bit Manipulation Instructions, Version 2                                                                                                                            |
+| AVX512VL           | AVX-512 Vector Length Extensions                                                                                                                                                   |
+| AVX512VNNI         | AVX-512 Vector Neural Network Instructions                                                                                                                                         |
+| AVX512VP2INTERSECT | AVX-512 Intersect for D/Q                                                                                                                                                          |
+| AVX512VPOPCNTDQ    | AVX-512 Vector Population Count Doubleword and Quadword                                                                                                                            |
+| AVXIFMA            | AVX-IFMA instructions                                                                                                                                                              |
+| AVXNECONVERT       | AVX-NE-CONVERT instructions                                                                                                                                                        |
+| AVXSLOW            | Indicates the CPU performs 2 128 bit operations instead of one                                                                                                                     |
+| AVXVNNI            | AVX (VEX encoded) VNNI neural network instructions                                                                                                                                 |
+| AVXVNNIINT8        | AVX-VNNI-INT8 instructions                                                                                                                                                         |
+| BMI1               | Bit Manipulation Instruction Set 1                                                                                                                                                 |
+| BMI2               | Bit Manipulation Instruction Set 2                                                                                                                                                 |
+| CETIBT             | Intel CET Indirect Branch Tracking                                                                                                                                                 |
+| CETSS              | Intel CET Shadow Stack                                                                                                                                                             |
+| CLDEMOTE           | Cache Line Demote                                                                                                                                                                  |
+| CLMUL              | Carry-less Multiplication                                                                                                                                                          |
+| CLZERO             | CLZERO instruction supported                                                                                                                                                       |
+| CMOV               | i686 CMOV                                                                                                                                                                          |
+| CMPCCXADD          | CMPCCXADD instructions                                                                                                                                                             |
+| CMPSB_SCADBS_SHORT | Fast short CMPSB and SCASB                                                                                                                                                         |
+| CMPXCHG8           | CMPXCHG8 instruction                                                                                                                                                               |
+| CPBOOST            | Core Performance Boost                                                                                                                                                             |
+| CPPC               | AMD: Collaborative Processor Performance Control                                                                                                                                   |
+| CX16               | CMPXCHG16B Instruction                                                                                                                                                             |
+| EFER_LMSLE_UNS     | AMD: =Core::X86::Msr::EFER[LMSLE] is not supported, and MBZ                                                                                                                        |
+| ENQCMD             | Enqueue Command                                                                                                                                                                    |
+| ERMS               | Enhanced REP MOVSB/STOSB                                                                                                                                                           |
+| F16C               | Half-precision floating-point conversion                                                                                                                                           |
+| FLUSH_L1D          | Flush L1D cache                                                                                                                                                                    |
+| FMA3               | Intel FMA 3. Does not imply AVX.                                                                                                                                                   |
+| FMA4               | Bulldozer FMA4 functions                                                                                                                                                           |
+| FP128              | AMD: When set, the internal FP/SIMD execution datapath is 128-bits wide                                                                                                            |
+| FP256              | AMD: When set, the internal FP/SIMD execution datapath is 256-bits wide                                                                                                            |
+| FSRM               | Fast Short Rep Mov                                                                                                                                                                 |
+| FXSR               | FXSAVE, FXRESTOR instructions, CR4 bit 9                                                                                                                                           |
+| FXSROPT            | FXSAVE/FXRSTOR optimizations                                                                                                                                                       |
+| GFNI               | Galois Field New Instructions. May require other features (AVX, AVX512VL,AVX512F) based on usage.                                                                                  |
+| HLE                | Hardware Lock Elision                                                                                                                                                              |
+| HRESET             | If set CPU supports history reset and the IA32_HRESET_ENABLE MSR                                                                                                                   |
+| HTT                | Hyperthreading (enabled)                                                                                                                                                           |
+| HWA                | Hardware assert supported. Indicates support for MSRC001_10                                                                                                                        |
+| HYBRID_CPU         | This part has CPUs of more than one type.                                                                                                                                          |
+| HYPERVISOR         | This bit has been reserved by Intel & AMD for use by hypervisors                                                                                                                   |
+| IA32_ARCH_CAP      | IA32_ARCH_CAPABILITIES MSR (Intel)                                                                                                                                                 |
+| IA32_CORE_CAP      | IA32_CORE_CAPABILITIES MSR                                                                                                                                                         |
+| IBPB               | Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB)                                                                                         |
+| IBRS               | AMD: Indirect Branch Restricted Speculation                                                                                                                                        |
+| IBRS_PREFERRED     | AMD: IBRS is preferred over software solution                                                                                                                                      |
+| IBRS_PROVIDES_SMP  | AMD: IBRS provides Same Mode Protection                                                                                                                                            |
+| IBS                | Instruction Based Sampling (AMD)                                                                                                                                                   |
+| IBSBRNTRGT         | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSFETCHSAM        | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSFFV             | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSOPCNT           | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSOPCNTEXT        | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSOPSAM           | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSRDWROPCNT       | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBSRIPINVALIDCHK   | Instruction Based Sampling Feature (AMD)                                                                                                                                           |
+| IBS_FETCH_CTLX     | AMD: IBS fetch control extended MSR supported                                                                                                                                      |
+| IBS_OPDATA4        | AMD: IBS op data 4 MSR supported                                                                                                                                                   |
+| IBS_OPFUSE         | AMD: Indicates support for IbsOpFuse                                                                                                                                               |
+| IBS_PREVENTHOST    | Disallowing IBS use by the host supported                                                                                                                                          |
+| IBS_ZEN4           | Fetch and Op IBS support IBS extensions added with Zen4                                                                                                                            |
+| INT_WBINVD         | WBINVD/WBNOINVD are interruptible.                                                                                                                                                 |
+| INVLPGB            | NVLPGB and TLBSYNC instruction supported                                                                                                                                           |
+| LAHF               | LAHF/SAHF in long mode                                                                                                                                                             |
+| LAM                | If set, CPU supports Linear Address Masking                                                                                                                                        |
+| LBRVIRT            | LBR virtualization                                                                                                                                                                 |
+| LZCNT              | LZCNT instruction                                                                                                                                                                  |
+| MCAOVERFLOW        | MCA overflow recovery support.                                                                                                                                                     |
+| MCDT_NO            | Processor do not exhibit MXCSR Configuration Dependent Timing behavior and do not need to mitigate it.                                                                             |
+| MCOMMIT            | MCOMMIT instruction supported                                                                                                                                                      |
+| MD_CLEAR           | VERW clears CPU buffers                                                                                                                                                            |
+| MMX                | standard MMX                                                                                                                                                                       |
+| MMXEXT             | SSE integer functions or AMD MMX ext                                                                                                                                               |
+| MOVBE              | MOVBE instruction (big-endian)                                                                                                                                                     |
+| MOVDIR64B          | Move 64 Bytes as Direct Store                                                                                                                                                      |
+| MOVDIRI            | Move Doubleword as Direct Store                                                                                                                                                    |
+| MOVSB_ZL           | Fast Zero-Length MOVSB                                                                                                                                                             |
+| MPX                | Intel MPX (Memory Protection Extensions)                                                                                                                                           |
+| MOVU               | MOVU SSE instructions are more efficient and should be preferred to SSE	MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD        |
+| MSRIRC             | Instruction Retired Counter MSR available                                                                                                                                          |
+| MSR_PAGEFLUSH      | Page Flush MSR available                                                                                                                                                           |
+| NRIPS              | Indicates support for NRIP save on VMEXIT                                                                                                                                          |
+| NX                 | NX (No-Execute) bit                                                                                                                                                                |
+| OSXSAVE            | XSAVE enabled by OS                                                                                                                                                                |
+| PCONFIG            | PCONFIG for Intel Multi-Key Total Memory Encryption                                                                                                                                |
+| POPCNT             | POPCNT instruction                                                                                                                                                                 |
+| PPIN               | AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled                                            |
+| PREFETCHI          | PREFETCHIT0/1 instructions                                            |
+| PSFD               | AMD: Predictive Store Forward Disable                                                                                                                                              |
+| RDPRU              | RDPRU instruction supported                                                                                                                                                        |
+| RDRAND             | RDRAND instruction is available                                                                                                                                                    |
+| RDSEED             | RDSEED instruction is available                                                                                                                                                    |
+| RDTSCP             | RDTSCP Instruction                                                                                                                                                                 |
+| RTM                | Restricted Transactional Memory                                                                                                                                                    |
+| RTM_ALWAYS_ABORT   | Indicates that the loaded microcode is forcing RTM abort.                                                                                                                          |
+| SERIALIZE          | Serialize Instruction Execution                                                                                                                                                    |
+| SEV                | AMD Secure Encrypted Virtualization supported                                                                                                                                      |
+| SEV_64BIT          | AMD SEV guest execution only allowed from a 64-bit host                                                                                                                            |
+| SEV_ALTERNATIVE    | AMD SEV Alternate Injection supported                                                                                                                                              |
+| SEV_DEBUGSWAP      | Full debug state swap supported for SEV-ES guests                                                                                                                                  |
+| SEV_ES             | AMD SEV Encrypted State supported                                                                                                                                                  |
+| SEV_RESTRICTED     | AMD SEV Restricted Injection supported                                                                                                                                             |
+| SEV_SNP            | AMD SEV Secure Nested Paging supported                                                                                                                                             |
+| SGX                | Software Guard Extensions                                                                                                                                                          |
+| SGXLC              | Software Guard Extensions Launch Control                                                                                                                                           |
+| SHA                | Intel SHA Extensions                                                                                                                                                               |
+| SME                | AMD Secure Memory Encryption supported                                                                                                                                             |
+| SME_COHERENT       | AMD Hardware cache coherency across encryption domains enforced                                                                                                                    |
+| SPEC_CTRL_SSBD     | Speculative Store Bypass Disable                                                                                                                                                   |
+| SRBDS_CTRL         | SRBDS mitigation MSR available                                                                                                                                                     |
+| SSE                | SSE functions                                                                                                                                                                      |
+| SSE2               | P4 SSE functions                                                                                                                                                                   |
+| SSE3               | Prescott SSE3 functions                                                                                                                                                            |
+| SSE4               | Penryn SSE4.1 functions                                                                                                                                                            |
+| SSE42              | Nehalem SSE4.2 functions                                                                                                                                                           |
+| SSE4A              | AMD Barcelona microarchitecture SSE4a instructions                                                                                                                                 |
+| SSSE3              | Conroe SSSE3 functions                                                                                                                                                             |
+| STIBP              | Single Thread Indirect Branch Predictors                                                                                                                                           |
+| STIBP_ALWAYSON     | AMD: Single Thread Indirect Branch Prediction Mode has Enhanced Performance and may be left Always On                                                                              |
+| STOSB_SHORT        | Fast short STOSB                                                                                                                                                                   |
+| SUCCOR             | Software uncorrectable error containment and recovery capability.                                                                                                                  |
+| SVM                | AMD Secure Virtual Machine                                                                                                                                                         |
+| SVMDA              | Indicates support for the SVM decode assists.                                                                                                                                      |
+| SVMFBASID          | SVM, Indicates that TLB flush events, including CR3 writes and CR4.PGE toggles, flush only the current ASID's TLB entries. Also indicates support for the extended VMCBTLB_Control |
+| SVML               | AMD SVM lock. Indicates support for SVM-Lock.                                                                                                                                      |
+| SVMNP              | AMD SVM nested paging                                                                                                                                                              |
+| SVMPF              | SVM pause intercept filter. Indicates support for the pause intercept filter                                                                                                       |
+| SVMPFT             | SVM PAUSE filter threshold. Indicates support for the PAUSE filter cycle count threshold                                                                                           |
+| SYSCALL            | System-Call Extension (SCE): SYSCALL and SYSRET instructions.                                                                                                                      |
+| SYSEE              | SYSENTER and SYSEXIT instructions                                                                                                                                                  |
+| TBM                | AMD Trailing Bit Manipulation                                                                                                                                                      |
+| TLB_FLUSH_NESTED   | AMD: Flushing includes all the nested translations for guest translations                                                                                                          |
+| TME                | Intel Total Memory Encryption. The following MSRs are supported: IA32_TME_CAPABILITY, IA32_TME_ACTIVATE, IA32_TME_EXCLUDE_MASK, and IA32_TME_EXCLUDE_BASE.                         |
+| TOPEXT             | TopologyExtensions: topology extensions support. Indicates support for CPUID Fn8000_001D_EAX_x[N:0]-CPUID Fn8000_001E_EDX.                                                         |
+| TSCRATEMSR         | MSR based TSC rate control. Indicates support for MSR TSC ratio MSRC000_0104                                                                                                       |
+| TSXLDTRK           | Intel TSX Suspend Load Address Tracking                                                                                                                                            |
+| VAES               | Vector AES. AVX(512) versions requires additional checks.                                                                                                                          |
+| VMCBCLEAN          | VMCB clean bits. Indicates support for VMCB clean bits.                                                                                                                            |
+| VMPL               | AMD VM Permission Levels supported                                                                                                                                                 |
+| VMSA_REGPROT       | AMD VMSA Register Protection supported                                                                                                                                             |
+| VMX                | Virtual Machine Extensions                                                                                                                                                         |
+| VPCLMULQDQ         | Carry-Less Multiplication Quadword. Requires AVX for 3 register versions.                                                                                                          |
+| VTE                | AMD Virtual Transparent Encryption supported                                                                                                                                       |
+| WAITPKG            | TPAUSE, UMONITOR, UMWAIT                                                                                                                                                           |
+| WBNOINVD           | Write Back and Do Not Invalidate Cache                                                                                                                                             |
+| X87                | FPU                                                                                                                                                                                |
+| XGETBV1            | Supports XGETBV with ECX = 1                                                                                                                                                       |
+| XOP                | Bulldozer XOP functions                                                                                                                                                            |
+| XSAVE              | XSAVE, XRESTOR, XSETBV, XGETBV                                                                                                                                                     |
+| XSAVEC             | Supports XSAVEC and the compacted form of XRSTOR.                                                                                                                                  |
+| XSAVEOPT           | XSAVEOPT available                                                                                                                                                                 |
+| XSAVES             | Supports XSAVES/XRSTORS and IA32_XSS                                                                                                                                               |
+
+# ARM features:
+
+| Feature Flag | Description                                                      |
+|--------------|------------------------------------------------------------------|
+| AESARM       | AES instructions                                                 |
+| ARMCPUID     | Some CPU ID registers readable at user-level                     |
+| ASIMD        | Advanced SIMD                                                    |
+| ASIMDDP      | SIMD Dot Product                                                 |
+| ASIMDHP      | Advanced SIMD half-precision floating point                      |
+| ASIMDRDM     | Rounding Double Multiply Accumulate/Subtract (SQRDMLAH/SQRDMLSH) |
+| ATOMICS      | Large System Extensions (LSE)                                    |
+| CRC32        | CRC32/CRC32C instructions                                        |
+| DCPOP        | Data cache clean to Point of Persistence (DC CVAP)               |
+| EVTSTRM      | Generic timer                                                    |
+| FCMA         | Floatin point complex number addition and multiplication         |
+| FP           | Single-precision and double-precision floating point             |
+| FPHP         | Half-precision floating point                                    |
+| GPA          | Generic Pointer Authentication                                   |
+| JSCVT        | Javascript-style double->int convert (FJCVTZS)                   |
+| LRCPC        | Weaker release consistency (LDAPR, etc)                          |
+| PMULL        | Polynomial Multiply instructions (PMULL/PMULL2)                  |
+| SHA1         | SHA-1 instructions (SHA1C, etc)                                  |
+| SHA2         | SHA-2 instructions (SHA256H, etc)                                |
+| SHA3         | SHA-3 instructions (EOR3, RAXI, XAR, BCAX)                       |
+| SHA512       | SHA512 instructions                                              |
+| SM3          | SM3 instructions                                                 |
+| SM4          | SM4 instructions                                                 |
+| SVE          | Scalable Vector Extension                                        |
+
 # license
 
 This code is published under an MIT license. See LICENSE file for more information.

vendor/github.com/klauspost/cpuid/v2/cpuid.go

@@ -73,6 +73,7 @@ const (
 	AMD3DNOW                            // AMD 3DNOW
 	AMD3DNOWEXT                         // AMD 3DNowExt
 	AMXBF16                             // Tile computational operations on BFLOAT16 numbers
+	AMXFP16                             // Tile computational operations on FP16 numbers
 	AMXINT8                             // Tile computational operations on 8-bit integers
 	AMXTILE                             // Tile architecture
 	AVX                                 // AVX functions
@@ -93,8 +94,11 @@ const (
 	AVX512VNNI                          // AVX-512 Vector Neural Network Instructions
 	AVX512VP2INTERSECT                  // AVX-512 Intersect for D/Q
 	AVX512VPOPCNTDQ                     // AVX-512 Vector Population Count Doubleword and Quadword
+	AVXIFMA                             // AVX-IFMA instructions
+	AVXNECONVERT                        // AVX-NE-CONVERT instructions
 	AVXSLOW                             // Indicates the CPU performs 2 128 bit operations instead of one
 	AVXVNNI                             // AVX (VEX encoded) VNNI neural network instructions
+	AVXVNNIINT8                         // AVX-VNNI-INT8 instructions
 	BMI1                                // Bit Manipulation Instruction Set 1
 	BMI2                                // Bit Manipulation Instruction Set 2
 	CETIBT                              // Intel CET Indirect Branch Tracking
@@ -103,16 +107,21 @@ const (
 	CLMUL                               // Carry-less Multiplication
 	CLZERO                              // CLZERO instruction supported
 	CMOV                                // i686 CMOV
+	CMPCCXADD                           // CMPCCXADD instructions
 	CMPSB_SCADBS_SHORT                  // Fast short CMPSB and SCASB
 	CMPXCHG8                            // CMPXCHG8 instruction
 	CPBOOST                             // Core Performance Boost
+	CPPC                                // AMD: Collaborative Processor Performance Control
 	CX16                                // CMPXCHG16B Instruction
+	EFER_LMSLE_UNS                      // AMD: =Core::X86::Msr::EFER[LMSLE] is not supported, and MBZ
 	ENQCMD                              // Enqueue Command
 	ERMS                                // Enhanced REP MOVSB/STOSB
 	F16C                                // Half-precision floating-point conversion
 	FLUSH_L1D                           // Flush L1D cache
 	FMA3                                // Intel FMA 3. Does not imply AVX.
 	FMA4                                // Bulldozer FMA4 functions
+	FP128                               // AMD: When set, the internal FP/SIMD execution datapath is no more than 128-bits wide
+	FP256                               // AMD: When set, the internal FP/SIMD execution datapath is no more than 256-bits wide
 	FSRM                                // Fast Short Rep Mov
 	FXSR                                // FXSAVE, FXRESTOR instructions, CR4 bit 9
 	FXSROPT                             // FXSAVE/FXRSTOR optimizations
@@ -126,6 +135,9 @@ const (
 	IA32_ARCH_CAP                       // IA32_ARCH_CAPABILITIES MSR (Intel)
 	IA32_CORE_CAP                       // IA32_CORE_CAPABILITIES MSR
 	IBPB                                // Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB)
+	IBRS                                // AMD: Indirect Branch Restricted Speculation
+	IBRS_PREFERRED                      // AMD: IBRS is preferred over software solution
+	IBRS_PROVIDES_SMP                   // AMD: IBRS provides Same Mode Protection
 	IBS                                 // Instruction Based Sampling (AMD)
 	IBSBRNTRGT                          // Instruction Based Sampling Feature (AMD)
 	IBSFETCHSAM                         // Instruction Based Sampling Feature (AMD)
@@ -135,7 +147,11 @@ const (
 	IBSOPSAM                            // Instruction Based Sampling Feature (AMD)
 	IBSRDWROPCNT                        // Instruction Based Sampling Feature (AMD)
 	IBSRIPINVALIDCHK                    // Instruction Based Sampling Feature (AMD)
+	IBS_FETCH_CTLX                      // AMD: IBS fetch control extended MSR supported
+	IBS_OPDATA4                         // AMD: IBS op data 4 MSR supported
+	IBS_OPFUSE                          // AMD: Indicates support for IbsOpFuse
 	IBS_PREVENTHOST                     // Disallowing IBS use by the host supported
+	IBS_ZEN4                            // AMD: Fetch and Op IBS support IBS extensions added with Zen4
 	INT_WBINVD                          // WBINVD/WBNOINVD are interruptible.
 	INVLPGB                             // NVLPGB and TLBSYNC instruction supported
 	LAHF                                // LAHF/SAHF in long mode
@@ -152,6 +168,7 @@ const (
 	MOVDIR64B                           // Move 64 Bytes as Direct Store
 	MOVDIRI                             // Move Doubleword as Direct Store
 	MOVSB_ZL                            // Fast Zero-Length MOVSB
+	MOVU                                // AMD: MOVU SSE instructions are more efficient and should be preferred to SSE	MOVL/MOVH. MOVUPS is more efficient than MOVLPS/MOVHPS. MOVUPD is more efficient than MOVLPD/MOVHPD
 	MPX                                 // Intel MPX (Memory Protection Extensions)
 	MSRIRC                              // Instruction Retired Counter MSR available
 	MSR_PAGEFLUSH                       // Page Flush MSR available
@@ -160,6 +177,9 @@ const (
 	OSXSAVE                             // XSAVE enabled by OS
 	PCONFIG                             // PCONFIG for Intel Multi-Key Total Memory Encryption
 	POPCNT                              // POPCNT instruction
+	PPIN                                // AMD: Protected Processor Inventory Number support. Indicates that Protected Processor Inventory Number (PPIN) capability can be enabled
+	PREFETCHI                           // PREFETCHIT0/1 instructions
+	PSFD                                // AMD: Predictive Store Forward Disable
 	RDPRU                               // RDPRU instruction supported
 	RDRAND                              // RDRAND instruction is available
 	RDSEED                              // RDSEED instruction is available
@@ -189,6 +209,7 @@ const (
 	SSE4A                               // AMD Barcelona microarchitecture SSE4a instructions
 	SSSE3                               // Conroe SSSE3 functions
 	STIBP                               // Single Thread Indirect Branch Predictors
+	STIBP_ALWAYSON                      // AMD: Single Thread Indirect Branch Prediction Mode has Enhanced Performance and may be left Always On
 	STOSB_SHORT                         // Fast short STOSB
 	SUCCOR                              // Software uncorrectable error containment and recovery capability.
 	SVM                                 // AMD Secure Virtual Machine
@@ -201,6 +222,7 @@ const (
 	SYSCALL                             // System-Call Extension (SCE): SYSCALL and SYSRET instructions.
 	SYSEE                               // SYSENTER and SYSEXIT instructions
 	TBM                                 // AMD Trailing Bit Manipulation
+	TLB_FLUSH_NESTED                    // AMD: Flushing includes all the nested translations for guest translations
 	TME                                 // Intel Total Memory Encryption. The following MSRs are supported: IA32_TME_CAPABILITY, IA32_TME_ACTIVATE, IA32_TME_EXCLUDE_MASK, and IA32_TME_EXCLUDE_BASE.
 	TOPEXT                              // TopologyExtensions: topology extensions support. Indicates support for CPUID Fn8000_001D_EAX_x[N:0]-CPUID Fn8000_001E_EDX.
 	TSCRATEMSR                          // MSR based TSC rate control. Indicates support for MSR TSC ratio MSRC000_0104
@@ -367,7 +389,7 @@ func (c CPUInfo) Supports(ids ...FeatureID) bool {
 
 // Has allows for checking a single feature.
 // Should be inlined by the compiler.
-func (c CPUInfo) Has(id FeatureID) bool {
+func (c *CPUInfo) Has(id FeatureID) bool {
 	return c.featureSet.inSet(id)
 }
 
@@ -381,26 +403,47 @@ func (c CPUInfo) AnyOf(ids ...FeatureID) bool {
 	return false
 }
 
+// Features contains several features combined for a fast check using
+// CpuInfo.HasAll
+type Features *flagSet
+
+// CombineFeatures allows to combine several features for a close to constant time lookup.
+func CombineFeatures(ids ...FeatureID) Features {
+	var v flagSet
+	for _, id := range ids {
+		v.set(id)
+	}
+	return &v
+}
+
+func (c *CPUInfo) HasAll(f Features) bool {
+	return c.featureSet.hasSetP(f)
+}
+
 // https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
-var level1Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2)
-var level2Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3)
-var level3Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE)
-var level4Features = flagSetWith(CMOV, CMPXCHG8, X87, FXSR, MMX, SYSCALL, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE, AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL)
+var oneOfLevel = CombineFeatures(SYSEE, SYSCALL)
+var level1Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2)
+var level2Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3)
+var level3Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE)
+var level4Features = CombineFeatures(CMOV, CMPXCHG8, X87, FXSR, MMX, SSE, SSE2, CX16, LAHF, POPCNT, SSE3, SSE4, SSE42, SSSE3, AVX, AVX2, BMI1, BMI2, F16C, FMA3, LZCNT, MOVBE, OSXSAVE, AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL)
 
 // X64Level returns the microarchitecture level detected on the CPU.
 // If features are lacking or non x64 mode, 0 is returned.
 // See https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
 func (c CPUInfo) X64Level() int {
-	if c.featureSet.hasSet(level4Features) {
+	if !c.featureSet.hasOneOf(oneOfLevel) {
+		return 0
+	}
+	if c.featureSet.hasSetP(level4Features) {
 		return 4
 	}
-	if c.featureSet.hasSet(level3Features) {
+	if c.featureSet.hasSetP(level3Features) {
 		return 3
 	}
-	if c.featureSet.hasSet(level2Features) {
+	if c.featureSet.hasSetP(level2Features) {
 		return 2
 	}
-	if c.featureSet.hasSet(level1Features) {
+	if c.featureSet.hasSetP(level1Features) {
 		return 1
 	}
 	return 0
@@ -564,7 +607,7 @@ const flagMask = flagBits - 1
 // flagSet contains detected cpu features and characteristics in an array of flags
 type flagSet [(lastID + flagMask) / flagBits]flags
 
-func (s flagSet) inSet(feat FeatureID) bool {
+func (s *flagSet) inSet(feat FeatureID) bool {
 	return s[feat>>flagBitsLog2]&(1<<(feat&flagMask)) != 0
 }
 
@@ -594,7 +637,7 @@ func (s *flagSet) or(other flagSet) {
 }
 
 // hasSet returns whether all features are present.
-func (s flagSet) hasSet(other flagSet) bool {
+func (s *flagSet) hasSet(other flagSet) bool {
 	for i, v := range other[:] {
 		if s[i]&v != v {
 			return false
@@ -603,8 +646,28 @@ func (s flagSet) hasSet(other flagSet) bool {
 	return true
 }
 
+// hasSet returns whether all features are present.
+func (s *flagSet) hasSetP(other *flagSet) bool {
+	for i, v := range other[:] {
+		if s[i]&v != v {
+			return false
+		}
+	}
+	return true
+}
+
+// hasOneOf returns whether one or more features are present.
+func (s *flagSet) hasOneOf(other *flagSet) bool {
+	for i, v := range other[:] {
+		if s[i]&v != 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // nEnabled will return the number of enabled flags.
-func (s flagSet) nEnabled() (n int) {
+func (s *flagSet) nEnabled() (n int) {
 	for _, v := range s[:] {
 		n += bits.OnesCount64(uint64(v))
 	}
@@ -1118,13 +1181,20 @@ func support() flagSet {
 		fs.setIf(edx&(1<<30) != 0, IA32_CORE_CAP)
 		fs.setIf(edx&(1<<31) != 0, SPEC_CTRL_SSBD)
 
-		// CPUID.(EAX=7, ECX=1)
+		// CPUID.(EAX=7, ECX=1).EDX
+		fs.setIf(edx&(1<<4) != 0, AVXVNNIINT8)
+		fs.setIf(edx&(1<<5) != 0, AVXNECONVERT)
+		fs.setIf(edx&(1<<14) != 0, PREFETCHI)
+
+		// CPUID.(EAX=7, ECX=1).EAX
 		eax1, _, _, _ := cpuidex(7, 1)
 		fs.setIf(fs.inSet(AVX) && eax1&(1<<4) != 0, AVXVNNI)
+		fs.setIf(eax1&(1<<7) != 0, CMPCCXADD)
 		fs.setIf(eax1&(1<<10) != 0, MOVSB_ZL)
 		fs.setIf(eax1&(1<<11) != 0, STOSB_SHORT)
 		fs.setIf(eax1&(1<<12) != 0, CMPSB_SCADBS_SHORT)
 		fs.setIf(eax1&(1<<22) != 0, HRESET)
+		fs.setIf(eax1&(1<<23) != 0, AVXIFMA)
 		fs.setIf(eax1&(1<<26) != 0, LAM)
 
 		// Only detect AVX-512 features if XGETBV is supported
@@ -1162,6 +1232,7 @@ func support() flagSet {
 				fs.setIf(edx&(1<<25) != 0, AMXINT8)
 				// eax1 = CPUID.(EAX=7, ECX=1).EAX
 				fs.setIf(eax1&(1<<5) != 0, AVX512BF16)
+				fs.setIf(eax1&(1<<21) != 0, AMXFP16)
 			}
 		}
 
@@ -1234,9 +1305,21 @@ func support() flagSet {
 
 	if maxExtendedFunction() >= 0x80000008 {
 		_, b, _, _ := cpuid(0x80000008)
+		fs.setIf(b&(1<<28) != 0, PSFD)
+		fs.setIf(b&(1<<27) != 0, CPPC)
+		fs.setIf(b&(1<<24) != 0, SPEC_CTRL_SSBD)
+		fs.setIf(b&(1<<23) != 0, PPIN)
+		fs.setIf(b&(1<<21) != 0, TLB_FLUSH_NESTED)
+		fs.setIf(b&(1<<20) != 0, EFER_LMSLE_UNS)
+		fs.setIf(b&(1<<19) != 0, IBRS_PROVIDES_SMP)
+		fs.setIf(b&(1<<18) != 0, IBRS_PREFERRED)
+		fs.setIf(b&(1<<17) != 0, STIBP_ALWAYSON)
+		fs.setIf(b&(1<<15) != 0, STIBP)
+		fs.setIf(b&(1<<14) != 0, IBRS)
+		fs.setIf((b&(1<<13)) != 0, INT_WBINVD)
+		fs.setIf(b&(1<<12) != 0, IBPB)
 		fs.setIf((b&(1<<9)) != 0, WBNOINVD)
 		fs.setIf((b&(1<<8)) != 0, MCOMMIT)
-		fs.setIf((b&(1<<13)) != 0, INT_WBINVD)
 		fs.setIf((b&(1<<4)) != 0, RDPRU)
 		fs.setIf((b&(1<<3)) != 0, INVLPGB)
 		fs.setIf((b&(1<<1)) != 0, MSRIRC)
@@ -1257,6 +1340,13 @@ func support() flagSet {
 		fs.setIf((edx>>12)&1 == 1, SVMPFT)
 	}
 
+	if maxExtendedFunction() >= 0x8000001a {
+		eax, _, _, _ := cpuid(0x8000001a)
+		fs.setIf((eax>>0)&1 == 1, FP128)
+		fs.setIf((eax>>1)&1 == 1, MOVU)
+		fs.setIf((eax>>2)&1 == 1, FP256)
+	}
+
 	if maxExtendedFunction() >= 0x8000001b && fs.inSet(IBS) {
 		eax, _, _, _ := cpuid(0x8000001b)
 		fs.setIf((eax>>0)&1 == 1, IBSFFV)
@@ -1267,6 +1357,10 @@ func support() flagSet {
 		fs.setIf((eax>>5)&1 == 1, IBSBRNTRGT)
 		fs.setIf((eax>>6)&1 == 1, IBSOPCNTEXT)
 		fs.setIf((eax>>7)&1 == 1, IBSRIPINVALIDCHK)
+		fs.setIf((eax>>8)&1 == 1, IBS_OPFUSE)
+		fs.setIf((eax>>9)&1 == 1, IBS_FETCH_CTLX)
+		fs.setIf((eax>>10)&1 == 1, IBS_OPDATA4) // Doc says "Fixed,0. IBS op data 4 MSR supported", but assuming they mean 1.
+		fs.setIf((eax>>11)&1 == 1, IBS_ZEN4)
 	}
 
 	if maxExtendedFunction() >= 0x8000001f && vend == AMD {

vendor/github.com/klauspost/cpuid/v2/featureid_string.go

@@ -13,185 +13,207 @@ func _() {
 	_ = x[AMD3DNOW-3]
 	_ = x[AMD3DNOWEXT-4]
 	_ = x[AMXBF16-5]
-	_ = x[AMXINT8-6]
-	_ = x[AMXTILE-7]
-	_ = x[AVX-8]
-	_ = x[AVX2-9]
-	_ = x[AVX512BF16-10]
-	_ = x[AVX512BITALG-11]
-	_ = x[AVX512BW-12]
-	_ = x[AVX512CD-13]
-	_ = x[AVX512DQ-14]
-	_ = x[AVX512ER-15]
-	_ = x[AVX512F-16]
-	_ = x[AVX512FP16-17]
-	_ = x[AVX512IFMA-18]
-	_ = x[AVX512PF-19]
-	_ = x[AVX512VBMI-20]
-	_ = x[AVX512VBMI2-21]
-	_ = x[AVX512VL-22]
-	_ = x[AVX512VNNI-23]
-	_ = x[AVX512VP2INTERSECT-24]
-	_ = x[AVX512VPOPCNTDQ-25]
-	_ = x[AVXSLOW-26]
-	_ = x[AVXVNNI-27]
-	_ = x[BMI1-28]
-	_ = x[BMI2-29]
-	_ = x[CETIBT-30]
-	_ = x[CETSS-31]
-	_ = x[CLDEMOTE-32]
-	_ = x[CLMUL-33]
-	_ = x[CLZERO-34]
-	_ = x[CMOV-35]
-	_ = x[CMPSB_SCADBS_SHORT-36]
-	_ = x[CMPXCHG8-37]
-	_ = x[CPBOOST-38]
-	_ = x[CX16-39]
-	_ = x[ENQCMD-40]
-	_ = x[ERMS-41]
-	_ = x[F16C-42]
-	_ = x[FLUSH_L1D-43]
-	_ = x[FMA3-44]
-	_ = x[FMA4-45]
-	_ = x[FSRM-46]
-	_ = x[FXSR-47]
-	_ = x[FXSROPT-48]
-	_ = x[GFNI-49]
-	_ = x[HLE-50]
-	_ = x[HRESET-51]
-	_ = x[HTT-52]
-	_ = x[HWA-53]
-	_ = x[HYBRID_CPU-54]
-	_ = x[HYPERVISOR-55]
-	_ = x[IA32_ARCH_CAP-56]
-	_ = x[IA32_CORE_CAP-57]
-	_ = x[IBPB-58]
-	_ = x[IBS-59]
-	_ = x[IBSBRNTRGT-60]
-	_ = x[IBSFETCHSAM-61]
-	_ = x[IBSFFV-62]
-	_ = x[IBSOPCNT-63]
-	_ = x[IBSOPCNTEXT-64]
-	_ = x[IBSOPSAM-65]
-	_ = x[IBSRDWROPCNT-66]
-	_ = x[IBSRIPINVALIDCHK-67]
-	_ = x[IBS_PREVENTHOST-68]
-	_ = x[INT_WBINVD-69]
-	_ = x[INVLPGB-70]
-	_ = x[LAHF-71]
-	_ = x[LAM-72]
-	_ = x[LBRVIRT-73]
-	_ = x[LZCNT-74]
-	_ = x[MCAOVERFLOW-75]
-	_ = x[MCDT_NO-76]
-	_ = x[MCOMMIT-77]
-	_ = x[MD_CLEAR-78]
-	_ = x[MMX-79]
-	_ = x[MMXEXT-80]
-	_ = x[MOVBE-81]
-	_ = x[MOVDIR64B-82]
-	_ = x[MOVDIRI-83]
-	_ = x[MOVSB_ZL-84]
-	_ = x[MPX-85]
-	_ = x[MSRIRC-86]
-	_ = x[MSR_PAGEFLUSH-87]
-	_ = x[NRIPS-88]
-	_ = x[NX-89]
-	_ = x[OSXSAVE-90]
-	_ = x[PCONFIG-91]
-	_ = x[POPCNT-92]
-	_ = x[RDPRU-93]
-	_ = x[RDRAND-94]
-	_ = x[RDSEED-95]
-	_ = x[RDTSCP-96]
-	_ = x[RTM-97]
-	_ = x[RTM_ALWAYS_ABORT-98]
-	_ = x[SERIALIZE-99]
-	_ = x[SEV-100]
-	_ = x[SEV_64BIT-101]
-	_ = x[SEV_ALTERNATIVE-102]
-	_ = x[SEV_DEBUGSWAP-103]
-	_ = x[SEV_ES-104]
-	_ = x[SEV_RESTRICTED-105]
-	_ = x[SEV_SNP-106]
-	_ = x[SGX-107]
-	_ = x[SGXLC-108]
-	_ = x[SHA-109]
-	_ = x[SME-110]
-	_ = x[SME_COHERENT-111]
-	_ = x[SPEC_CTRL_SSBD-112]
-	_ = x[SRBDS_CTRL-113]
-	_ = x[SSE-114]
-	_ = x[SSE2-115]
-	_ = x[SSE3-116]
-	_ = x[SSE4-117]
-	_ = x[SSE42-118]
-	_ = x[SSE4A-119]
-	_ = x[SSSE3-120]
-	_ = x[STIBP-121]
-	_ = x[STOSB_SHORT-122]
-	_ = x[SUCCOR-123]
-	_ = x[SVM-124]
-	_ = x[SVMDA-125]
-	_ = x[SVMFBASID-126]
-	_ = x[SVML-127]
-	_ = x[SVMNP-128]
-	_ = x[SVMPF-129]
-	_ = x[SVMPFT-130]
-	_ = x[SYSCALL-131]
-	_ = x[SYSEE-132]
-	_ = x[TBM-133]
-	_ = x[TME-134]
-	_ = x[TOPEXT-135]
-	_ = x[TSCRATEMSR-136]
-	_ = x[TSXLDTRK-137]
-	_ = x[VAES-138]
-	_ = x[VMCBCLEAN-139]
-	_ = x[VMPL-140]
-	_ = x[VMSA_REGPROT-141]
-	_ = x[VMX-142]
-	_ = x[VPCLMULQDQ-143]
-	_ = x[VTE-144]
-	_ = x[WAITPKG-145]
-	_ = x[WBNOINVD-146]
-	_ = x[X87-147]
-	_ = x[XGETBV1-148]
-	_ = x[XOP-149]
-	_ = x[XSAVE-150]
-	_ = x[XSAVEC-151]
-	_ = x[XSAVEOPT-152]
-	_ = x[XSAVES-153]
-	_ = x[AESARM-154]
-	_ = x[ARMCPUID-155]
-	_ = x[ASIMD-156]
-	_ = x[ASIMDDP-157]
-	_ = x[ASIMDHP-158]
-	_ = x[ASIMDRDM-159]
-	_ = x[ATOMICS-160]
-	_ = x[CRC32-161]
-	_ = x[DCPOP-162]
-	_ = x[EVTSTRM-163]
-	_ = x[FCMA-164]
-	_ = x[FP-165]
-	_ = x[FPHP-166]
-	_ = x[GPA-167]
-	_ = x[JSCVT-168]
-	_ = x[LRCPC-169]
-	_ = x[PMULL-170]
-	_ = x[SHA1-171]
-	_ = x[SHA2-172]
-	_ = x[SHA3-173]
-	_ = x[SHA512-174]
-	_ = x[SM3-175]
-	_ = x[SM4-176]
-	_ = x[SVE-177]
-	_ = x[lastID-178]
+	_ = x[AMXFP16-6]
+	_ = x[AMXINT8-7]
+	_ = x[AMXTILE-8]
+	_ = x[AVX-9]
+	_ = x[AVX2-10]
+	_ = x[AVX512BF16-11]
+	_ = x[AVX512BITALG-12]
+	_ = x[AVX512BW-13]
+	_ = x[AVX512CD-14]
+	_ = x[AVX512DQ-15]
+	_ = x[AVX512ER-16]
+	_ = x[AVX512F-17]
+	_ = x[AVX512FP16-18]
+	_ = x[AVX512IFMA-19]
+	_ = x[AVX512PF-20]
+	_ = x[AVX512VBMI-21]
+	_ = x[AVX512VBMI2-22]
+	_ = x[AVX512VL-23]
+	_ = x[AVX512VNNI-24]
+	_ = x[AVX512VP2INTERSECT-25]
+	_ = x[AVX512VPOPCNTDQ-26]
+	_ = x[AVXIFMA-27]
+	_ = x[AVXNECONVERT-28]
+	_ = x[AVXSLOW-29]
+	_ = x[AVXVNNI-30]
+	_ = x[AVXVNNIINT8-31]
+	_ = x[BMI1-32]
+	_ = x[BMI2-33]
+	_ = x[CETIBT-34]
+	_ = x[CETSS-35]
+	_ = x[CLDEMOTE-36]
+	_ = x[CLMUL-37]
+	_ = x[CLZERO-38]
+	_ = x[CMOV-39]
+	_ = x[CMPCCXADD-40]
+	_ = x[CMPSB_SCADBS_SHORT-41]
+	_ = x[CMPXCHG8-42]
+	_ = x[CPBOOST-43]
+	_ = x[CPPC-44]
+	_ = x[CX16-45]
+	_ = x[EFER_LMSLE_UNS-46]
+	_ = x[ENQCMD-47]
+	_ = x[ERMS-48]
+	_ = x[F16C-49]
+	_ = x[FLUSH_L1D-50]
+	_ = x[FMA3-51]
+	_ = x[FMA4-52]
+	_ = x[FP128-53]
+	_ = x[FP256-54]
+	_ = x[FSRM-55]
+	_ = x[FXSR-56]
+	_ = x[FXSROPT-57]
+	_ = x[GFNI-58]
+	_ = x[HLE-59]
+	_ = x[HRESET-60]
+	_ = x[HTT-61]
+	_ = x[HWA-62]
+	_ = x[HYBRID_CPU-63]
+	_ = x[HYPERVISOR-64]
+	_ = x[IA32_ARCH_CAP-65]
+	_ = x[IA32_CORE_CAP-66]
+	_ = x[IBPB-67]
+	_ = x[IBRS-68]
+	_ = x[IBRS_PREFERRED-69]
+	_ = x[IBRS_PROVIDES_SMP-70]
+	_ = x[IBS-71]
+	_ = x[IBSBRNTRGT-72]
+	_ = x[IBSFETCHSAM-73]
+	_ = x[IBSFFV-74]
+	_ = x[IBSOPCNT-75]
+	_ = x[IBSOPCNTEXT-76]
+	_ = x[IBSOPSAM-77]
+	_ = x[IBSRDWROPCNT-78]
+	_ = x[IBSRIPINVALIDCHK-79]
+	_ = x[IBS_FETCH_CTLX-80]
+	_ = x[IBS_OPDATA4-81]
+	_ = x[IBS_OPFUSE-82]
+	_ = x[IBS_PREVENTHOST-83]
+	_ = x[IBS_ZEN4-84]
+	_ = x[INT_WBINVD-85]
+	_ = x[INVLPGB-86]
+	_ = x[LAHF-87]
+	_ = x[LAM-88]
+	_ = x[LBRVIRT-89]
+	_ = x[LZCNT-90]
+	_ = x[MCAOVERFLOW-91]
+	_ = x[MCDT_NO-92]
+	_ = x[MCOMMIT-93]
+	_ = x[MD_CLEAR-94]
+	_ = x[MMX-95]
+	_ = x[MMXEXT-96]
+	_ = x[MOVBE-97]
+	_ = x[MOVDIR64B-98]
+	_ = x[MOVDIRI-99]
+	_ = x[MOVSB_ZL-100]
+	_ = x[MOVU-101]
+	_ = x[MPX-102]
+	_ = x[MSRIRC-103]
+	_ = x[MSR_PAGEFLUSH-104]
+	_ = x[NRIPS-105]
+	_ = x[NX-106]
+	_ = x[OSXSAVE-107]
+	_ = x[PCONFIG-108]
+	_ = x[POPCNT-109]
+	_ = x[PPIN-110]
+	_ = x[PREFETCHI-111]
+	_ = x[PSFD-112]
+	_ = x[RDPRU-113]
+	_ = x[RDRAND-114]
+	_ = x[RDSEED-115]
+	_ = x[RDTSCP-116]
+	_ = x[RTM-117]
+	_ = x[RTM_ALWAYS_ABORT-118]
+	_ = x[SERIALIZE-119]
+	_ = x[SEV-120]
+	_ = x[SEV_64BIT-121]
+	_ = x[SEV_ALTERNATIVE-122]
+	_ = x[SEV_DEBUGSWAP-123]
+	_ = x[SEV_ES-124]
+	_ = x[SEV_RESTRICTED-125]
+	_ = x[SEV_SNP-126]
+	_ = x[SGX-127]
+	_ = x[SGXLC-128]
+	_ = x[SHA-129]
+	_ = x[SME-130]
+	_ = x[SME_COHERENT-131]
+	_ = x[SPEC_CTRL_SSBD-132]
+	_ = x[SRBDS_CTRL-133]
+	_ = x[SSE-134]
+	_ = x[SSE2-135]
+	_ = x[SSE3-136]
+	_ = x[SSE4-137]
+	_ = x[SSE42-138]
+	_ = x[SSE4A-139]
+	_ = x[SSSE3-140]
+	_ = x[STIBP-141]
+	_ = x[STIBP_ALWAYSON-142]
+	_ = x[STOSB_SHORT-143]
+	_ = x[SUCCOR-144]
+	_ = x[SVM-145]
+	_ = x[SVMDA-146]
+	_ = x[SVMFBASID-147]
+	_ = x[SVML-148]
+	_ = x[SVMNP-149]
+	_ = x[SVMPF-150]
+	_ = x[SVMPFT-151]
+	_ = x[SYSCALL-152]
+	_ = x[SYSEE-153]
+	_ = x[TBM-154]
+	_ = x[TLB_FLUSH_NESTED-155]
+	_ = x[TME-156]
+	_ = x[TOPEXT-157]
+	_ = x[TSCRATEMSR-158]
+	_ = x[TSXLDTRK-159]
+	_ = x[VAES-160]
+	_ = x[VMCBCLEAN-161]
+	_ = x[VMPL-162]
+	_ = x[VMSA_REGPROT-163]
+	_ = x[VMX-164]
+	_ = x[VPCLMULQDQ-165]
+	_ = x[VTE-166]
+	_ = x[WAITPKG-167]
+	_ = x[WBNOINVD-168]
+	_ = x[X87-169]
+	_ = x[XGETBV1-170]
+	_ = x[XOP-171]
+	_ = x[XSAVE-172]
+	_ = x[XSAVEC-173]
+	_ = x[XSAVEOPT-174]
+	_ = x[XSAVES-175]
+	_ = x[AESARM-176]
+	_ = x[ARMCPUID-177]
+	_ = x[ASIMD-178]
+	_ = x[ASIMDDP-179]
+	_ = x[ASIMDHP-180]
+	_ = x[ASIMDRDM-181]
+	_ = x[ATOMICS-182]
+	_ = x[CRC32-183]
+	_ = x[DCPOP-184]
+	_ = x[EVTSTRM-185]
+	_ = x[FCMA-186]
+	_ = x[FP-187]
+	_ = x[FPHP-188]
+	_ = x[GPA-189]
+	_ = x[JSCVT-190]
+	_ = x[LRCPC-191]
+	_ = x[PMULL-192]
+	_ = x[SHA1-193]
+	_ = x[SHA2-194]
+	_ = x[SHA3-195]
+	_ = x[SHA512-196]
+	_ = x[SM3-197]
+	_ = x[SM4-198]
+	_ = x[SVE-199]
+	_ = x[lastID-200]
 	_ = x[firstID-0]
 }
 
-const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXSLOWAVXVNNIBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCX16ENQCMDERMSF16CFLUSH_L1DFMA3FMA4FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_PREVENTHOSTINT_WBINVDINVLPGBLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMPXMSRIRCMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTRDPRURDRANDRDSEEDRDTSCPRTMRTM_ALWAYS_ABORTSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
+const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXTILEAVXAVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8BMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4INT_WBINVDINVLPGBLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRTMRTM_ALWAYS_ABORTSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
 
-var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 58, 62, 72, 84, 92, 100, 108, 116, 123, 133, 143, 151, 161, 172, 180, 190, 208, 223, 230, 237, 241, 245, 251, 256, 264, 269, 275, 279, 297, 305, 312, 316, 322, 326, 330, 339, 343, 347, 351, 355, 362, 366, 369, 375, 378, 381, 391, 401, 414, 427, 431, 434, 444, 455, 461, 469, 480, 488, 500, 516, 531, 541, 548, 552, 555, 562, 567, 578, 585, 592, 600, 603, 609, 614, 623, 630, 638, 641, 647, 660, 665, 667, 674, 681, 687, 692, 698, 704, 710, 713, 729, 738, 741, 750, 765, 778, 784, 798, 805, 808, 813, 816, 819, 831, 845, 855, 858, 862, 866, 870, 875, 880, 885, 890, 901, 907, 910, 915, 924, 928, 933, 938, 944, 951, 956, 959, 962, 968, 978, 986, 990, 999, 1003, 1015, 1018, 1028, 1031, 1038, 1046, 1049, 1056, 1059, 1064, 1070, 1078, 1084, 1090, 1098, 1103, 1110, 1117, 1125, 1132, 1137, 1142, 1149, 1153, 1155, 1159, 1162, 1167, 1172, 1177, 1181, 1185, 1189, 1195, 1198, 1201, 1204, 1210}
+var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 62, 65, 69, 79, 91, 99, 107, 115, 123, 130, 140, 150, 158, 168, 179, 187, 197, 215, 230, 237, 249, 256, 263, 274, 278, 282, 288, 293, 301, 306, 312, 316, 325, 343, 351, 358, 362, 366, 380, 386, 390, 394, 403, 407, 411, 416, 421, 425, 429, 436, 440, 443, 449, 452, 455, 465, 475, 488, 501, 505, 509, 523, 540, 543, 553, 564, 570, 578, 589, 597, 609, 625, 639, 650, 660, 675, 683, 693, 700, 704, 707, 714, 719, 730, 737, 744, 752, 755, 761, 766, 775, 782, 790, 794, 797, 803, 816, 821, 823, 830, 837, 843, 847, 856, 860, 865, 871, 877, 883, 886, 902, 911, 914, 923, 938, 951, 957, 971, 978, 981, 986, 989, 992, 1004, 1018, 1028, 1031, 1035, 1039, 1043, 1048, 1053, 1058, 1063, 1077, 1088, 1094, 1097, 1102, 1111, 1115, 1120, 1125, 1131, 1138, 1143, 1146, 1162, 1165, 1171, 1181, 1189, 1193, 1202, 1206, 1218, 1221, 1231, 1234, 1241, 1249, 1252, 1259, 1262, 1267, 1273, 1281, 1287, 1293, 1301, 1306, 1313, 1320, 1328, 1335, 1340, 1345, 1352, 1356, 1358, 1362, 1365, 1370, 1375, 1380, 1384, 1388, 1392, 1398, 1401, 1404, 1407, 1413}
 
 func (i FeatureID) String() string {
 	if i < 0 || i >= FeatureID(len(_FeatureID_index)-1) {

vendor/github.com/klauspost/cpuid/v2/os_darwin_arm64.go

@@ -83,7 +83,7 @@ func tryToFillCPUInfoFomSysctl(c *CPUInfo) {
 	c.Model = sysctlGetInt(0, "machdep.cpu.model")
 	c.CacheLine = sysctlGetInt64(0, "hw.cachelinesize")
 	c.Cache.L1I = sysctlGetInt64(-1, "hw.l1icachesize")
-	c.Cache.L1D = sysctlGetInt64(-1, "hw.l1icachesize")
+	c.Cache.L1D = sysctlGetInt64(-1, "hw.l1dcachesize")
 	c.Cache.L2 = sysctlGetInt64(-1, "hw.l2cachesize")
 	c.Cache.L3 = sysctlGetInt64(-1, "hw.l3cachesize")
 

vendor/github.com/mattn/go-isatty/isatty_bsd.go

@@ -1,5 +1,5 @@
-//go:build (darwin || freebsd || openbsd || netbsd || dragonfly) && !appengine
-// +build darwin freebsd openbsd netbsd dragonfly
+//go:build (darwin || freebsd || openbsd || netbsd || dragonfly || hurd) && !appengine
+// +build darwin freebsd openbsd netbsd dragonfly hurd
 // +build !appengine
 
 package isatty

vendor/github.com/minio/minio-go/v7/Makefile

@@ -16,6 +16,8 @@ lint:
 
 vet:
 	@GO111MODULE=on go vet ./...
+	@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
+	${GOPATH}/bin/staticcheck -tests=false -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-ST1023,-ST1005"
 
 test:
 	@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./...

vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go

@@ -21,7 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/xml"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 
@@ -143,5 +143,5 @@ func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]b
 		}
 	}
 
-	return ioutil.ReadAll(resp.Body)
+	return io.ReadAll(resp.Body)
 }

vendor/github.com/minio/minio-go/v7/api-bucket-policy.go

@@ -18,7 +18,7 @@ package minio
 
 import (
 	"context"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -137,7 +137,7 @@ func (c *Client) getBucketPolicy(ctx context.Context, bucketName string) (string
 		}
 	}
 
-	bucketPolicyBuf, err := ioutil.ReadAll(resp.Body)
+	bucketPolicyBuf, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
 	}

vendor/github.com/minio/minio-go/v7/api-bucket-replication.go

@@ -22,7 +22,7 @@ import (
 	"context"
 	"encoding/json"
 	"encoding/xml"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"time"
@@ -180,7 +180,7 @@ func (c *Client) GetBucketReplicationMetrics(ctx context.Context, bucketName str
 	if resp.StatusCode != http.StatusOK {
 		return s, httpRespToErrorResponse(resp, bucketName, "")
 	}
-	respBytes, err := ioutil.ReadAll(resp.Body)
+	respBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return s, err
 	}

vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go

@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -58,7 +57,7 @@ func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags
 		return nil, httpRespToErrorResponse(resp, bucketName, "")
 	}
 
-	defer io.Copy(ioutil.Discard, resp.Body)
+	defer io.Copy(io.Discard, resp.Body)
 	return tags.ParseBucketXML(resp.Body)
 }
 

vendor/github.com/minio/minio-go/v7/api-compose-object.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -221,7 +220,7 @@ func (c *Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuc
 		headers.Set(minIOBucketSourceETag, dstOpts.Internal.SourceETag)
 	}
 	if dstOpts.Internal.ReplicationRequest {
-		headers.Set(minIOBucketReplicationRequest, "")
+		headers.Set(minIOBucketReplicationRequest, "true")
 	}
 	if !dstOpts.Internal.LegalholdTimestamp.IsZero() {
 		headers.Set(minIOBucketReplicationObjectLegalHoldTimestamp, dstOpts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano))
@@ -516,7 +515,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
 				return UploadInfo{}, err
 			}
 			if dst.Progress != nil {
-				io.CopyN(ioutil.Discard, dst.Progress, end-start+1)
+				io.CopyN(io.Discard, dst.Progress, end-start+1)
 			}
 			objParts = append(objParts, complPart)
 			partIndex++
@@ -525,7 +524,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
 
 	// 4. Make final complete-multipart request.
 	uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID,
-		completeMultipartUpload{Parts: objParts}, PutObjectOptions{})
+		completeMultipartUpload{Parts: objParts}, PutObjectOptions{ServerSideEncryption: dst.Encryption})
 	if err != nil {
 		return UploadInfo{}, err
 	}

vendor/github.com/minio/minio-go/v7/api-copy-object.go

@@ -20,7 +20,6 @@ package minio
 import (
 	"context"
 	"io"
-	"io/ioutil"
 	"net/http"
 )
 
@@ -54,7 +53,7 @@ func (c *Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySr
 
 	// Update the progress properly after successful copy.
 	if dst.Progress != nil {
-		io.Copy(ioutil.Discard, io.LimitReader(dst.Progress, dst.Size))
+		io.Copy(io.Discard, io.LimitReader(dst.Progress, dst.Size))
 	}
 
 	cpObjRes := copyObjectResult{}

vendor/github.com/minio/minio-go/v7/api-datatypes.go

@@ -87,6 +87,8 @@ type UploadInfo struct {
 	ExpirationRuleID string
 
 	// Verified checksum values, if any.
+	// Values are base64 (standard) encoded.
+	// For multipart objects this is a checksum of the checksum of each part.
 	ChecksumCRC32  string
 	ChecksumCRC32C string
 	ChecksumSHA1   string
@@ -147,7 +149,8 @@ type ObjectInfo struct {
 	// - FAILED
 	// - REPLICA (on the destination)
 	ReplicationStatus string `xml:"ReplicationStatus"`
-
+	// set to true if delete marker has backing object version on target, and eligible to replicate
+	ReplicationReady bool
 	// Lifecycle expiry-date and ruleID associated with the expiry
 	// not to be confused with `Expires` HTTP header.
 	Expiration       time.Time

vendor/github.com/minio/minio-go/v7/api-error-response.go

@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 )
 
@@ -108,7 +107,7 @@ const (
 func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) {
 	// read the whole body (up to 1MB)
 	const maxBodyLength = 1 << 20
-	body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
+	body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
 	if err != nil {
 		return nil, err
 	}
@@ -253,26 +252,6 @@ func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string)
 	}
 }
 
-// errInvalidBucketName - Invalid bucket name response.
-func errInvalidBucketName(message string) error {
-	return ErrorResponse{
-		StatusCode: http.StatusBadRequest,
-		Code:       "InvalidBucketName",
-		Message:    message,
-		RequestID:  "minio",
-	}
-}
-
-// errInvalidObjectName - Invalid object name response.
-func errInvalidObjectName(message string) error {
-	return ErrorResponse{
-		StatusCode: http.StatusNotFound,
-		Code:       "NoSuchKey",
-		Message:    message,
-		RequestID:  "minio",
-	}
-}
-
 // errInvalidArgument - Invalid argument response.
 func errInvalidArgument(message string) error {
 	return ErrorResponse{

vendor/github.com/minio/minio-go/v7/api-get-options.go

@@ -27,8 +27,9 @@ import (
 
 // AdvancedGetOptions for internal use by MinIO server - not intended for client use.
 type AdvancedGetOptions struct {
-	ReplicationDeleteMarker bool
-	ReplicationProxyRequest string
+	ReplicationDeleteMarker           bool
+	IsReplicationReadyForDeleteMarker bool
+	ReplicationProxyRequest           string
 }
 
 // GetObjectOptions are used to specify additional headers or options

vendor/github.com/minio/minio-go/v7/api-list.go

@@ -70,21 +70,28 @@ func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts List
 	// Return object owner information by default
 	fetchOwner := true
 
+	sendObjectInfo := func(info ObjectInfo) {
+		select {
+		case objectStatCh <- info:
+		case <-ctx.Done():
+		}
+	}
+
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		defer close(objectStatCh)
-		objectStatCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return objectStatCh
 	}
 
 	// Validate incoming object prefix.
 	if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
 		defer close(objectStatCh)
-		objectStatCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return objectStatCh
 	}
 
@@ -98,9 +105,9 @@ func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts List
 			result, err := c.listObjectsV2Query(ctx, bucketName, opts.Prefix, continuationToken,
 				fetchOwner, opts.WithMetadata, delimiter, opts.StartAfter, opts.MaxKeys, opts.headers)
 			if err != nil {
-				objectStatCh <- ObjectInfo{
+				sendObjectInfo(ObjectInfo{
 					Err: err,
-				}
+				})
 				return
 			}
 
@@ -137,6 +144,14 @@ func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts List
 			if !result.IsTruncated {
 				return
 			}
+
+			// Add this to catch broken S3 API implementations.
+			if continuationToken == "" {
+				sendObjectInfo(ObjectInfo{
+					Err: fmt.Errorf("listObjectsV2 is truncated without continuationToken, %s S3 server is incompatible with S3 API", c.endpointURL),
+				})
+				return
+			}
 		}
 	}(objectStatCh)
 	return objectStatCh
@@ -262,20 +277,28 @@ func (c *Client) listObjects(ctx context.Context, bucketName string, opts ListOb
 		// If recursive we do not delimit.
 		delimiter = ""
 	}
+
+	sendObjectInfo := func(info ObjectInfo) {
+		select {
+		case objectStatCh <- info:
+		case <-ctx.Done():
+		}
+	}
+
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		defer close(objectStatCh)
-		objectStatCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return objectStatCh
 	}
 	// Validate incoming object prefix.
 	if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
 		defer close(objectStatCh)
-		objectStatCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return objectStatCh
 	}
 
@@ -288,9 +311,9 @@ func (c *Client) listObjects(ctx context.Context, bucketName string, opts ListOb
 			// Get list of objects a maximum of 1000 per request.
 			result, err := c.listObjectsQuery(ctx, bucketName, opts.Prefix, marker, delimiter, opts.MaxKeys, opts.headers)
 			if err != nil {
-				objectStatCh <- ObjectInfo{
+				sendObjectInfo(ObjectInfo{
 					Err: err,
-				}
+				})
 				return
 			}
 
@@ -343,21 +366,28 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
 		delimiter = ""
 	}
 
+	sendObjectInfo := func(info ObjectInfo) {
+		select {
+		case resultCh <- info:
+		case <-ctx.Done():
+		}
+	}
+
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		defer close(resultCh)
-		resultCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return resultCh
 	}
 
 	// Validate incoming object prefix.
 	if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
 		defer close(resultCh)
-		resultCh <- ObjectInfo{
+		sendObjectInfo(ObjectInfo{
 			Err: err,
-		}
+		})
 		return resultCh
 	}
 
@@ -374,9 +404,9 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
 			// Get list of objects a maximum of 1000 per request.
 			result, err := c.listObjectVersionsQuery(ctx, bucketName, opts.Prefix, keyMarker, versionIDMarker, delimiter, opts.MaxKeys, opts.headers)
 			if err != nil {
-				resultCh <- ObjectInfo{
+				sendObjectInfo(ObjectInfo{
 					Err: err,
-				}
+				})
 				return
 			}
 
@@ -867,6 +897,8 @@ func (c *Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyM
 }
 
 // listObjectParts list all object parts recursively.
+//
+//lint:ignore U1000 Keep this around
 func (c *Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) {
 	// Part number marker for the next batch of request.
 	var nextPartNumberMarker int

vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go

@@ -26,7 +26,6 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"sort"
@@ -159,8 +158,9 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
 			crcBytes = append(crcBytes, cSum...)
 		}
 
+		p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, sha256Hex: sha256Hex, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
 		// Proceed to upload the part.
-		objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption, !opts.DisableContentSha256, customHeader)
+		objPart, uerr := c.uploadPart(ctx, p)
 		if uerr != nil {
 			return UploadInfo{}, uerr
 		}
@@ -200,7 +200,9 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
 
 	// Sort all completed parts.
 	sort.Sort(completedParts(complMultipartUpload.Parts))
-	opts = PutObjectOptions{}
+	opts = PutObjectOptions{
+		ServerSideEncryption: opts.ServerSideEncryption,
+	}
 	if len(crcBytes) > 0 {
 		// Add hash of hashes.
 		crc.Reset()
@@ -269,57 +271,73 @@ func (c *Client) initiateMultipartUpload(ctx context.Context, bucketName, object
 	return initiateMultipartUploadResult, nil
 }
 
+type uploadPartParams struct {
+	bucketName   string
+	objectName   string
+	uploadID     string
+	reader       io.Reader
+	partNumber   int
+	md5Base64    string
+	sha256Hex    string
+	size         int64
+	sse          encrypt.ServerSide
+	streamSha256 bool
+	customHeader http.Header
+	trailer      http.Header
+}
+
 // uploadPart - Uploads a part in a multipart upload.
-func (c *Client) uploadPart(ctx context.Context, bucketName string, objectName string, uploadID string, reader io.Reader, partNumber int, md5Base64 string, sha256Hex string, size int64, sse encrypt.ServerSide, streamSha256 bool, customHeader http.Header) (ObjectPart, error) {
+func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart, error) {
 	// Input validation.
-	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
+	if err := s3utils.CheckValidBucketName(p.bucketName); err != nil {
 		return ObjectPart{}, err
 	}
-	if err := s3utils.CheckValidObjectName(objectName); err != nil {
+	if err := s3utils.CheckValidObjectName(p.objectName); err != nil {
 		return ObjectPart{}, err
 	}
-	if size > maxPartSize {
-		return ObjectPart{}, errEntityTooLarge(size, maxPartSize, bucketName, objectName)
+	if p.size > maxPartSize {
+		return ObjectPart{}, errEntityTooLarge(p.size, maxPartSize, p.bucketName, p.objectName)
 	}
-	if size <= -1 {
-		return ObjectPart{}, errEntityTooSmall(size, bucketName, objectName)
+	if p.size <= -1 {
+		return ObjectPart{}, errEntityTooSmall(p.size, p.bucketName, p.objectName)
 	}
-	if partNumber <= 0 {
+	if p.partNumber <= 0 {
 		return ObjectPart{}, errInvalidArgument("Part number cannot be negative or equal to zero.")
 	}
-	if uploadID == "" {
+	if p.uploadID == "" {
 		return ObjectPart{}, errInvalidArgument("UploadID cannot be empty.")
 	}
 
 	// Get resources properly escaped and lined up before using them in http request.
 	urlValues := make(url.Values)
 	// Set part number.
-	urlValues.Set("partNumber", strconv.Itoa(partNumber))
+	urlValues.Set("partNumber", strconv.Itoa(p.partNumber))
 	// Set upload id.
-	urlValues.Set("uploadId", uploadID)
+	urlValues.Set("uploadId", p.uploadID)
 
 	// Set encryption headers, if any.
-	if customHeader == nil {
-		customHeader = make(http.Header)
+	if p.customHeader == nil {
+		p.customHeader = make(http.Header)
 	}
 	// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
 	// Server-side encryption is supported by the S3 Multipart Upload actions.
 	// Unless you are using a customer-provided encryption key, you don't need
 	// to specify the encryption parameters in each UploadPart request.
-	if sse != nil && sse.Type() == encrypt.SSEC {
-		sse.Marshal(customHeader)
+	if p.sse != nil && p.sse.Type() == encrypt.SSEC {
+		p.sse.Marshal(p.customHeader)
 	}
 
 	reqMetadata := requestMetadata{
-		bucketName:       bucketName,
-		objectName:       objectName,
+		bucketName:       p.bucketName,
+		objectName:       p.objectName,
 		queryValues:      urlValues,
-		customHeader:     customHeader,
-		contentBody:      reader,
-		contentLength:    size,
-		contentMD5Base64: md5Base64,
-		contentSHA256Hex: sha256Hex,
-		streamSha256:     streamSha256,
+		customHeader:     p.customHeader,
+		contentBody:      p.reader,
+		contentLength:    p.size,
+		contentMD5Base64: p.md5Base64,
+		contentSHA256Hex: p.sha256Hex,
+		streamSha256:     p.streamSha256,
+		trailer:          p.trailer,
 	}
 
 	// Execute PUT on each part.
@@ -330,7 +348,7 @@ func (c *Client) uploadPart(ctx context.Context, bucketName string, objectName s
 	}
 	if resp != nil {
 		if resp.StatusCode != http.StatusOK {
-			return ObjectPart{}, httpRespToErrorResponse(resp, bucketName, objectName)
+			return ObjectPart{}, httpRespToErrorResponse(resp, p.bucketName, p.objectName)
 		}
 	}
 	// Once successfully uploaded, return completed part.
@@ -341,8 +359,8 @@ func (c *Client) uploadPart(ctx context.Context, bucketName string, objectName s
 		ChecksumSHA1:   h.Get("x-amz-checksum-sha1"),
 		ChecksumSHA256: h.Get("x-amz-checksum-sha256"),
 	}
-	objPart.Size = size
-	objPart.PartNumber = partNumber
+	objPart.Size = p.size
+	objPart.PartNumber = p.partNumber
 	// Trim off the odd double quotes from ETag in the beginning and end.
 	objPart.ETag = trimEtag(h.Get("ETag"))
 	return objPart, nil
@@ -395,7 +413,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
 
 	// Read resp.Body into a []bytes to parse for Error response inside the body
 	var b []byte
-	b, err = ioutil.ReadAll(resp.Body)
+	b, err = io.ReadAll(resp.Body)
 	if err != nil {
 		return UploadInfo{}, err
 	}
@@ -431,5 +449,10 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
 		Location:         completeMultipartUploadResult.Location,
 		Expiration:       expTime,
 		ExpirationRuleID: ruleID,
+
+		ChecksumSHA256: completeMultipartUploadResult.ChecksumSHA256,
+		ChecksumSHA1:   completeMultipartUploadResult.ChecksumSHA1,
+		ChecksumCRC32:  completeMultipartUploadResult.ChecksumCRC32,
+		ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
 	}, nil
 }

vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -44,7 +45,9 @@ import (
 func (c *Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string,
 	reader io.Reader, size int64, opts PutObjectOptions,
 ) (info UploadInfo, err error) {
-	if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 {
+	if opts.ConcurrentStreamParts && opts.NumThreads > 1 {
+		info, err = c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts)
+	} else if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 {
 		// Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader.
 		info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts)
 	} else {
@@ -107,11 +110,19 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
 		return UploadInfo{}, err
 	}
 
+	withChecksum := c.trailingHeaderSupport
+	if withChecksum {
+		if opts.UserMetadata == nil {
+			opts.UserMetadata = make(map[string]string, 1)
+		}
+		opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C"
+	}
 	// Initiate a new multipart upload.
 	uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
 	if err != nil {
 		return UploadInfo{}, err
 	}
+	delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
 
 	// Aborts the multipart upload in progress, if the
 	// function returns any error, since we do not resume
@@ -177,14 +188,33 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
 				// As a special case if partNumber is lastPartNumber, we
 				// calculate the offset based on the last part size.
 				if uploadReq.PartNum == lastPartNumber {
-					readOffset = (size - lastPartSize)
+					readOffset = size - lastPartSize
 					partSize = lastPartSize
 				}
 
 				sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress)
+				var trailer = make(http.Header, 1)
+				if withChecksum {
+					crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+					trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(crc.Sum(nil)))
+					sectionReader = newHashReaderWrapper(sectionReader, crc, func(hash []byte) {
+						trailer.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(hash))
+					})
+				}
 
 				// Proceed to upload the part.
-				objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption, !opts.DisableContentSha256, nil)
+				p := uploadPartParams{bucketName: bucketName,
+					objectName:   objectName,
+					uploadID:     uploadID,
+					reader:       sectionReader,
+					partNumber:   uploadReq.PartNum,
+					size:         partSize,
+					sse:          opts.ServerSideEncryption,
+					streamSha256: !opts.DisableContentSha256,
+					sha256Hex:    "",
+					trailer:      trailer,
+				}
+				objPart, err := c.uploadPart(ctx, p)
 				if err != nil {
 					uploadedPartsCh <- uploadedPartRes{
 						Error: err,
@@ -221,8 +251,12 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
 			// Update the totalUploadedSize.
 			totalUploadedSize += uploadRes.Size
 			complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
-				ETag:       uploadRes.Part.ETag,
-				PartNumber: uploadRes.Part.PartNumber,
+				ETag:           uploadRes.Part.ETag,
+				PartNumber:     uploadRes.Part.PartNumber,
+				ChecksumCRC32:  uploadRes.Part.ChecksumCRC32,
+				ChecksumCRC32C: uploadRes.Part.ChecksumCRC32C,
+				ChecksumSHA1:   uploadRes.Part.ChecksumSHA1,
+				ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
 			})
 		}
 	}
@@ -235,7 +269,22 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
 	// Sort all completed parts.
 	sort.Sort(completedParts(complMultipartUpload.Parts))
 
-	uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
+	opts = PutObjectOptions{
+		ServerSideEncryption: opts.ServerSideEncryption,
+	}
+	if withChecksum {
+		// Add hash of hashes.
+		crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+		for _, part := range complMultipartUpload.Parts {
+			cs, err := base64.StdEncoding.DecodeString(part.ChecksumCRC32C)
+			if err == nil {
+				crc.Write(cs)
+			}
+		}
+		opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
+	}
+
+	uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
 	if err != nil {
 		return UploadInfo{}, err
 	}
@@ -339,8 +388,8 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
 		// Update progress reader appropriately to the latest offset
 		// as we read from the source.
 		hooked := newHook(bytes.NewReader(buf[:length]), opts.Progress)
-
-		objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, hooked, partNumber, md5Base64, "", partSize, opts.ServerSideEncryption, !opts.DisableContentSha256, customHeader)
+		p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: hooked, partNumber: partNumber, md5Base64: md5Base64, size: partSize, sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
+		objPart, uerr := c.uploadPart(ctx, p)
 		if uerr != nil {
 			return UploadInfo{}, uerr
 		}
@@ -382,6 +431,211 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
 	// Sort all completed parts.
 	sort.Sort(completedParts(complMultipartUpload.Parts))
 
+	opts = PutObjectOptions{
+		ServerSideEncryption: opts.ServerSideEncryption,
+	}
+	if len(crcBytes) > 0 {
+		// Add hash of hashes.
+		crc.Reset()
+		crc.Write(crcBytes)
+		opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
+	}
+	uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
+	if err != nil {
+		return UploadInfo{}, err
+	}
+
+	uploadInfo.Size = totalUploadedSize
+	return uploadInfo, nil
+}
+
+// putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel.
+// This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer.
+func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string,
+	reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) {
+	// Input validation.
+	if err = s3utils.CheckValidBucketName(bucketName); err != nil {
+		return UploadInfo{}, err
+	}
+
+	if err = s3utils.CheckValidObjectName(objectName); err != nil {
+		return UploadInfo{}, err
+	}
+
+	if !opts.SendContentMd5 {
+		if opts.UserMetadata == nil {
+			opts.UserMetadata = make(map[string]string, 1)
+		}
+		opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C"
+	}
+
+	// Cancel all when an error occurs.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// Calculate the optimal parts info for a given size.
+	totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize)
+	if err != nil {
+		return UploadInfo{}, err
+	}
+
+	// Initiates a new multipart request
+	uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
+	if err != nil {
+		return UploadInfo{}, err
+	}
+	delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
+
+	// Aborts the multipart upload if the function returns
+	// any error, since we do not resume we should purge
+	// the parts which have been uploaded to relinquish
+	// storage space.
+	defer func() {
+		if err != nil {
+			c.abortMultipartUpload(ctx, bucketName, objectName, uploadID)
+		}
+	}()
+
+	// Create checksums
+	// CRC32C is ~50% faster on AMD64 @ 30GB/s
+	var crcBytes []byte
+	crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+	md5Hash := c.md5Hasher()
+	defer md5Hash.Close()
+
+	// Total data read and written to server. should be equal to 'size' at the end of the call.
+	var totalUploadedSize int64
+
+	// Initialize parts uploaded map.
+	partsInfo := make(map[int]ObjectPart)
+
+	// Create a buffer.
+	nBuffers := int64(opts.NumThreads)
+	bufs := make(chan []byte, nBuffers)
+	all := make([]byte, nBuffers*partSize)
+	for i := int64(0); i < nBuffers; i++ {
+		bufs <- all[i*partSize : i*partSize+partSize]
+	}
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	errCh := make(chan error, opts.NumThreads)
+
+	reader = newHook(reader, opts.Progress)
+
+	// Part number always starts with '1'.
+	var partNumber int
+	for partNumber = 1; partNumber <= totalPartsCount; partNumber++ {
+		// Proceed to upload the part.
+		var buf []byte
+		select {
+		case buf = <-bufs:
+		case err = <-errCh:
+			cancel()
+			wg.Wait()
+			return UploadInfo{}, err
+		}
+
+		if int64(len(buf)) != partSize {
+			return UploadInfo{}, fmt.Errorf("read buffer < %d than expected partSize: %d", len(buf), partSize)
+		}
+
+		length, rerr := readFull(reader, buf)
+		if rerr == io.EOF && partNumber > 1 {
+			// Done
+			break
+		}
+
+		if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF {
+			cancel()
+			wg.Wait()
+			return UploadInfo{}, rerr
+		}
+
+		// Calculate md5sum.
+		customHeader := make(http.Header)
+		if !opts.SendContentMd5 {
+			// Add CRC32C instead.
+			crc.Reset()
+			crc.Write(buf[:length])
+			cSum := crc.Sum(nil)
+			customHeader.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(cSum))
+			crcBytes = append(crcBytes, cSum...)
+		}
+
+		wg.Add(1)
+		go func(partNumber int) {
+			// Avoid declaring variables in the for loop
+			var md5Base64 string
+
+			if opts.SendContentMd5 {
+				md5Hash.Reset()
+				md5Hash.Write(buf[:length])
+				md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil))
+			}
+
+			defer wg.Done()
+			p := uploadPartParams{
+				bucketName:   bucketName,
+				objectName:   objectName,
+				uploadID:     uploadID,
+				reader:       bytes.NewReader(buf[:length]),
+				partNumber:   partNumber,
+				md5Base64:    md5Base64,
+				size:         int64(length),
+				sse:          opts.ServerSideEncryption,
+				streamSha256: !opts.DisableContentSha256,
+				customHeader: customHeader,
+			}
+			objPart, uerr := c.uploadPart(ctx, p)
+			if uerr != nil {
+				errCh <- uerr
+			}
+
+			// Save successfully uploaded part metadata.
+			mu.Lock()
+			partsInfo[partNumber] = objPart
+			mu.Unlock()
+
+			// Send buffer back so it can be reused.
+			bufs <- buf
+		}(partNumber)
+
+		// Save successfully uploaded size.
+		totalUploadedSize += int64(length)
+	}
+	wg.Wait()
+
+	// Collect any error
+	select {
+	case err = <-errCh:
+		return UploadInfo{}, err
+	default:
+	}
+
+	// Complete multipart upload.
+	var complMultipartUpload completeMultipartUpload
+
+	// Loop over total uploaded parts to save them in
+	// Parts array before completing the multipart request.
+	for i := 1; i < partNumber; i++ {
+		part, ok := partsInfo[i]
+		if !ok {
+			return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
+		}
+		complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
+			ETag:           part.ETag,
+			PartNumber:     part.PartNumber,
+			ChecksumCRC32:  part.ChecksumCRC32,
+			ChecksumCRC32C: part.ChecksumCRC32C,
+			ChecksumSHA1:   part.ChecksumSHA1,
+			ChecksumSHA256: part.ChecksumSHA256,
+		})
+	}
+
+	// Sort all completed parts.
+	sort.Sort(completedParts(complMultipartUpload.Parts))
+
 	opts = PutObjectOptions{}
 	if len(crcBytes) > 0 {
 		// Add hash of hashes.

vendor/github.com/minio/minio-go/v7/api-put-object.go

@@ -87,7 +87,12 @@ type PutObjectOptions struct {
 	SendContentMd5          bool
 	DisableContentSha256    bool
 	DisableMultipart        bool
-	Internal                AdvancedPutOptions
+
+	// ConcurrentStreamParts will create NumThreads buffers of PartSize bytes,
+	// fill them serially and upload them in parallel.
+	// This can be used for faster uploads on non-seekable or slow-to-seek input.
+	ConcurrentStreamParts bool
+	Internal              AdvancedPutOptions
 }
 
 // getNumThreads - gets the number of threads to be used in the multipart
@@ -159,7 +164,7 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 		header.Set(minIOBucketSourceETag, opts.Internal.SourceETag)
 	}
 	if opts.Internal.ReplicationRequest {
-		header.Set(minIOBucketReplicationRequest, "")
+		header.Set(minIOBucketReplicationRequest, "true")
 	}
 	if !opts.Internal.LegalholdTimestamp.IsZero() {
 		header.Set(minIOBucketReplicationObjectLegalHoldTimestamp, opts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano))
@@ -269,6 +274,12 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str
 	}
 
 	if size < 0 {
+		if opts.DisableMultipart {
+			return UploadInfo{}, errors.New("no length provided and multipart disabled")
+		}
+		if opts.ConcurrentStreamParts && opts.NumThreads > 1 {
+			return c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts)
+		}
 		return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts)
 	}
 
@@ -366,7 +377,8 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
 		rd := newHook(bytes.NewReader(buf[:length]), opts.Progress)
 
 		// Proceed to upload the part.
-		objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, "", int64(length), opts.ServerSideEncryption, !opts.DisableContentSha256, customHeader)
+		p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}
+		objPart, uerr := c.uploadPart(ctx, p)
 		if uerr != nil {
 			return UploadInfo{}, uerr
 		}

vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go

@@ -24,7 +24,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"sync"
@@ -107,7 +106,7 @@ func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts
 			return nopReadSeekCloser{bytes.NewReader(b.Bytes())}, int64(b.Len()), nil
 		}
 	} else {
-		f, err := ioutil.TempFile("", "s3-putsnowballobjects-*")
+		f, err := os.CreateTemp("", "s3-putsnowballobjects-*")
 		if err != nil {
 			return err
 		}

vendor/github.com/minio/minio-go/v7/api-remove.go

@@ -166,7 +166,7 @@ func (c *Client) removeObject(ctx context.Context, bucketName, objectName string
 		headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
 	}
 	if opts.Internal.ReplicationRequest {
-		headers.Set(minIOBucketReplicationRequest, "")
+		headers.Set(minIOBucketReplicationRequest, "true")
 	}
 	if opts.ForceDelete {
 		headers.Set(minIOForceDelete, "true")

vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go

@@ -316,17 +316,15 @@ type completeMultipartUploadResult struct {
 // CompletePart sub container lists individual part numbers and their
 // md5sum, part of completeMultipartUpload.
 type CompletePart struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"`
-
 	// Part number identifies the part.
 	PartNumber int
 	ETag       string
 
 	// Checksum values
-	ChecksumCRC32  string
-	ChecksumCRC32C string
-	ChecksumSHA1   string
-	ChecksumSHA256 string
+	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
+	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
+	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
+	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
 }
 
 // completeMultipartUpload container for completing multipart upload.

vendor/github.com/minio/minio-go/v7/api-select.go

@@ -41,8 +41,8 @@ type CSVFileHeaderInfo string
 // Constants for file header info.
 const (
 	CSVFileHeaderInfoNone   CSVFileHeaderInfo = "NONE"
-	CSVFileHeaderInfoIgnore                   = "IGNORE"
-	CSVFileHeaderInfoUse                      = "USE"
+	CSVFileHeaderInfoIgnore CSVFileHeaderInfo = "IGNORE"
+	CSVFileHeaderInfoUse    CSVFileHeaderInfo = "USE"
 )
 
 // SelectCompressionType - is the parameter for what type of compression is
@@ -52,15 +52,15 @@ type SelectCompressionType string
 // Constants for compression types under select API.
 const (
 	SelectCompressionNONE SelectCompressionType = "NONE"
-	SelectCompressionGZIP                       = "GZIP"
-	SelectCompressionBZIP                       = "BZIP2"
+	SelectCompressionGZIP SelectCompressionType = "GZIP"
+	SelectCompressionBZIP SelectCompressionType = "BZIP2"
 
 	// Non-standard compression schemes, supported by MinIO hosts:
 
-	SelectCompressionZSTD   = "ZSTD"   // Zstandard compression.
-	SelectCompressionLZ4    = "LZ4"    // LZ4 Stream
-	SelectCompressionS2     = "S2"     // S2 Stream
-	SelectCompressionSNAPPY = "SNAPPY" // Snappy stream
+	SelectCompressionZSTD   SelectCompressionType = "ZSTD"   // Zstandard compression.
+	SelectCompressionLZ4    SelectCompressionType = "LZ4"    // LZ4 Stream
+	SelectCompressionS2     SelectCompressionType = "S2"     // S2 Stream
+	SelectCompressionSNAPPY SelectCompressionType = "SNAPPY" // Snappy stream
 )
 
 // CSVQuoteFields - is the parameter for how CSV fields are quoted.
@@ -69,7 +69,7 @@ type CSVQuoteFields string
 // Constants for csv quote styles.
 const (
 	CSVQuoteFieldsAlways   CSVQuoteFields = "Always"
-	CSVQuoteFieldsAsNeeded                = "AsNeeded"
+	CSVQuoteFieldsAsNeeded CSVQuoteFields = "AsNeeded"
 )
 
 // QueryExpressionType - is of what syntax the expression is, this should only
@@ -87,7 +87,7 @@ type JSONType string
 // Constants for JSONTypes.
 const (
 	JSONDocumentType JSONType = "DOCUMENT"
-	JSONLinesType             = "LINES"
+	JSONLinesType    JSONType = "LINES"
 )
 
 // ParquetInputOptions parquet input specific options
@@ -378,8 +378,8 @@ type SelectObjectType string
 // Constants for input data types.
 const (
 	SelectObjectTypeCSV     SelectObjectType = "CSV"
-	SelectObjectTypeJSON                     = "JSON"
-	SelectObjectTypeParquet                  = "Parquet"
+	SelectObjectTypeJSON    SelectObjectType = "JSON"
+	SelectObjectTypeParquet SelectObjectType = "Parquet"
 )
 
 // preludeInfo is used for keeping track of necessary information from the
@@ -416,7 +416,7 @@ type messageType string
 
 const (
 	errorMsg  messageType = "error"
-	commonMsg             = "event"
+	commonMsg messageType = "event"
 )
 
 // eventType represents the type of event.
@@ -425,9 +425,9 @@ type eventType string
 // list of event-types returned by Select API.
 const (
 	endEvent      eventType = "End"
-	recordsEvent            = "Records"
-	progressEvent           = "Progress"
-	statsEvent              = "Stats"
+	recordsEvent  eventType = "Records"
+	progressEvent eventType = "Progress"
+	statsEvent    eventType = "Stats"
 )
 
 // contentType represents content type of event.