From 9ad19fbdd62310cbc6bdfdc1362a5840c52d7b5a Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Tue, 8 Nov 2022 14:44:47 +0100 Subject: [PATCH] Fix reading partial config If the config on the disk doesn't have all fields, then the missing fields are now populated with their defaults. --- config/store/fixtures/config_v1.json | 138 ++++++++++++++++++++ config/store/fixtures/config_v1_v3.json | 163 +++++++++++++++++++++++ config/store/fixtures/config_v2.json | 140 ++++++++++++++++++++ config/store/fixtures/config_v2_v3.json | 165 ++++++++++++++++++++++++ config/store/json.go | 96 ++++++++------ config/store/json_test.go | 50 +++++++ config/value/auth0.go | 1 + config/value/network.go | 8 ++ config/value/os.go | 5 + config/value/primitives.go | 7 + config/value/time.go | 1 + 11 files changed, 734 insertions(+), 40 deletions(-) create mode 100644 config/store/fixtures/config_v1.json create mode 100644 config/store/fixtures/config_v1_v3.json create mode 100644 config/store/fixtures/config_v2.json create mode 100644 config/store/fixtures/config_v2_v3.json create mode 100644 config/store/json_test.go diff --git a/config/store/fixtures/config_v1.json b/config/store/fixtures/config_v1.json new file mode 100644 index 00000000..b85a216c --- /dev/null +++ b/config/store/fixtures/config_v1.json @@ -0,0 +1,138 @@ +{ + "created_at": "2022-11-08T12:01:22.533279+01:00", + "version": 1, + "id": "c5ea4473-2f84-417c-a0c6-35746bfc9fc9", + "name": "cool-breeze-4646", + "address": ":8080", + "update_check": true, + "log": { + "level": "info", + "topics": [], + "max_lines": 1000 + }, + "db": { + "dir": "./config" + }, + "host": { + "name": [], + "auto": true + }, + "api": { + "read_only": false, + "access": { + "http": { + "allow": [], + "block": [] + }, + "https": { + "allow": [], + "block": [] + } + }, + "auth": { + "enable": false, + "disable_localhost": false, + "username": "", + "password": "", + "jwt": { + "secret": "L(*C[:uuHzL.]Fzpk$q=fa@PO=Z;j;56" + }, + "auth0": { + "enable": false, + "tenants": [] + } + } + }, + "tls": { + "address": ":8181", + "enable": false, + "auto": false, + "cert_file": "", + "key_file": "" + }, + "storage": { + "disk": { + "dir": "./data", + "max_size_mbytes": 0, + "cache": { + "enable": true, + "max_size_mbytes": 0, + "ttl_seconds": 300, + "max_file_size_mbytes": 1, + "types": [] + } + }, + "memory": { + "auth": { + "enable": true, + "username": "admin", + "password": "dcFsZVGwVFkv1bE8Rl" + }, + "max_size_mbytes": 0, + "purge": false + }, + "cors": { + "origins": [ + "*" + ] + }, + "mimetypes_file": "./mime.types" + }, + "ffmpeg": { + "binary": "ffmpeg", + "max_processes": 0, + "access": { + "input": { + "allow": [], + "block": [] + }, + "output": { + "allow": [], + "block": [] + } + }, + "log": { + "max_lines": 50, + "max_history": 3 + } + }, + "playout": { + "enable": false, + "min_port": 0, + "max_port": 0 + }, + "debug": { + "profiling": false, + "force_gc": 0 + }, + "metrics": { + "enable": false, + "enable_prometheus": false, + "range_sec": 300, + "interval_sec": 2 + }, + "sessions": { + "enable": true, + "ip_ignorelist": [ + "127.0.0.1/32", + "::1/128" + ], + "session_timeout_sec": 30, + "persist": false, + "persist_interval_sec": 300, + "max_bitrate_mbit": 0, + "max_sessions": 0 + }, + "service": { + "enable": false, + "token": "", + "url": "https://service.datarhei.com" + }, + "router": { + "blocked_prefixes": [ + "/api" + ], + "routes": {}, + "ui_path": "" + } +} \ No newline at end of file diff --git a/config/store/fixtures/config_v1_v3.json b/config/store/fixtures/config_v1_v3.json new file mode 100644 index 00000000..398673d8 --- /dev/null +++ b/config/store/fixtures/config_v1_v3.json @@ -0,0 +1,163 @@ +{ + "created_at": "2022-11-08T13:34:47.498911+01:00", + "version": 3, + "id": "c5ea4473-2f84-417c-a0c6-35746bfc9fc9", + "name": "cool-breeze-4646", + "address": ":8080", + "update_check": true, + "log": { + "level": "info", + "topics": [], + "max_lines": 1000 + }, + "db": { + "dir": "./config" + }, + "host": { + "name": [], + "auto": true + }, + "api": { + "read_only": false, + "access": { + "http": { + "allow": [], + "block": [] + }, + "https": { + "allow": [], + "block": [] + } + }, + "auth": { + "enable": false, + "disable_localhost": false, + "username": "", + "password": "", + "jwt": { + "secret": "L(*C[:uuHzL.]Fzpk$q=fa@PO=Z;j;56" + }, + "auth0": { + "enable": false, + "tenants": [] + } + } + }, + "tls": { + "address": ":8181", + "enable": false, + "auto": false, + "email": "cert@datarhei.com", + "cert_file": "", + "key_file": "" + }, + "storage": { + "disk": { + "dir": "./data", + "max_size_mbytes": 0, + "cache": { + "enable": true, + "max_size_mbytes": 0, + "ttl_seconds": 300, + "max_file_size_mbytes": 1, + "types": { + "allow": [], + "block": [ + ".m3u8", + ".mpd" + ] + } + } + }, + "memory": { + "auth": { + "enable": true, + "username": "admin", + "password": "dcFsZVGwVFkv1bE8Rl" + }, + "max_size_mbytes": 0, + "purge": false + }, + "cors": { + "origins": [ + "*" + ] + }, + "mimetypes_file": "./mime.types" + }, + "rtmp": { + "enable": false, + "enable_tls": false, + "address": ":1935", + "address_tls": ":1936", + "app": "/", + "token": "" + }, + "srt": { + "enable": false, + "address": ":6000", + "passphrase": "", + "token": "", + "log": { + "enable": false, + "topics": [] + } + }, + "ffmpeg": { + "binary": "ffmpeg", + "max_processes": 0, + "access": { + "input": { + "allow": [], + "block": [] + }, + "output": { + "allow": [], + "block": [] + } + }, + "log": { + "max_lines": 50, + "max_history": 3 + } + }, + "playout": { + "enable": false, + "min_port": 0, + "max_port": 0 + }, + "debug": { + "profiling": false, + "force_gc": 0 + }, + "metrics": { + "enable": false, + "enable_prometheus": false, + "range_sec": 300, + "interval_sec": 2 + }, + "sessions": { + "enable": true, + "ip_ignorelist": [ + "127.0.0.1/32", + "::1/128" + ], + "session_timeout_sec": 30, + "persist": false, + "persist_interval_sec": 300, + "max_bitrate_mbit": 0, + "max_sessions": 0 + }, + "service": { + "enable": false, + "token": "", + "url": "https://service.datarhei.com" + }, + "router": { + "blocked_prefixes": [ + "/api" + ], + "routes": {}, + "ui_path": "" + } +} \ No newline at end of file diff --git a/config/store/fixtures/config_v2.json b/config/store/fixtures/config_v2.json new file mode 100644 index 00000000..c6536b37 --- /dev/null +++ b/config/store/fixtures/config_v2.json @@ -0,0 +1,140 @@ +{ + "created_at": "2022-11-08T11:54:44.224213+01:00", + "version": 2, + "id": "3bddc061-e534-4315-ab56-95b48c050ec9", + "name": "super-frog-1715", + "address": ":8080", + "update_check": true, + "log": { + "level": "info", + "topics": [], + "max_lines": 1000 + }, + "db": { + "dir": "./config" + }, + "host": { + "name": [], + "auto": true + }, + "api": { + "read_only": false, + "access": { + "http": { + "allow": [], + "block": [] + }, + "https": { + "allow": [], + "block": [] + } + }, + "auth": { + "enable": false, + "disable_localhost": false, + "username": "", + "password": "", + "jwt": { + "secret": "u4+N,UDq]jGxGbbQLQN[!jcMsa\u0026weIJW" + }, + "auth0": { + "enable": false, + "tenants": [] + } + } + }, + "tls": { + "address": ":8181", + "enable": false, + "auto": false, + "cert_file": "", + "key_file": "" + }, + "storage": { + "disk": { + "dir": "./data", + "max_size_mbytes": 0, + "cache": { + "enable": true, + "max_size_mbytes": 0, + "ttl_seconds": 300, + "max_file_size_mbytes": 1, + "types": [ + ".ts" + ] + } + }, + "memory": { + "auth": { + "enable": true, + "username": "admin", + "password": "DsAKRUg9wmOk4qpvvy" + }, + "max_size_mbytes": 0, + "purge": false + }, + "cors": { + "origins": [ + "*" + ] + }, + "mimetypes_file": "./mime.types" + }, + "ffmpeg": { + "binary": "ffmpeg", + "max_processes": 0, + "access": { + "input": { + "allow": [], + "block": [] + }, + "output": { + "allow": [], + "block": [] + } + }, + "log": { + "max_lines": 50, + "max_history": 3 + } + }, + "playout": { + "enable": false, + "min_port": 0, + "max_port": 0 + }, + "debug": { + "profiling": false, + "force_gc": 0 + }, + "metrics": { + "enable": false, + "enable_prometheus": false, + "range_sec": 300, + "interval_sec": 2 + }, + "sessions": { + "enable": true, + "ip_ignorelist": [ + "127.0.0.1/32", + "::1/128" + ], + "session_timeout_sec": 30, + "persist": false, + "persist_interval_sec": 300, + "max_bitrate_mbit": 0, + "max_sessions": 0 + }, + "service": { + "enable": false, + "token": "", + "url": "https://service.datarhei.com" + }, + "router": { + "blocked_prefixes": [ + "/api" + ], + "routes": {}, + "ui_path": "" + } +} \ No newline at end of file diff --git a/config/store/fixtures/config_v2_v3.json b/config/store/fixtures/config_v2_v3.json new file mode 100644 index 00000000..0661f638 --- /dev/null +++ b/config/store/fixtures/config_v2_v3.json @@ -0,0 +1,165 @@ +{ + "created_at": "2022-11-08T11:54:44.224213+01:00", + "version": 3, + "id": "3bddc061-e534-4315-ab56-95b48c050ec9", + "name": "super-frog-1715", + "address": ":8080", + "update_check": true, + "log": { + "level": "info", + "topics": [], + "max_lines": 1000 + }, + "db": { + "dir": "./config" + }, + "host": { + "name": [], + "auto": true + }, + "api": { + "read_only": false, + "access": { + "http": { + "allow": [], + "block": [] + }, + "https": { + "allow": [], + "block": [] + } + }, + "auth": { + "enable": false, + "disable_localhost": false, + "username": "", + "password": "", + "jwt": { + "secret": "u4+N,UDq]jGxGbbQLQN[!jcMsa\u0026weIJW" + }, + "auth0": { + "enable": false, + "tenants": [] + } + } + }, + "tls": { + "address": ":8181", + "enable": false, + "auto": false, + "cert_file": "", + "key_file": "", + "email": "cert@datarhei.com" + }, + "storage": { + "disk": { + "dir": "./data", + "max_size_mbytes": 0, + "cache": { + "enable": true, + "max_size_mbytes": 0, + "ttl_seconds": 300, + "max_file_size_mbytes": 1, + "types": { + "allow": [ + ".ts" + ], + "block": [ + ".m3u8", + ".mpd" + ] + } + } + }, + "memory": { + "auth": { + "enable": true, + "username": "admin", + "password": "DsAKRUg9wmOk4qpvvy" + }, + "max_size_mbytes": 0, + "purge": false + }, + "cors": { + "origins": [ + "*" + ] + }, + "mimetypes_file": "./mime.types" + }, + "rtmp": { + "enable": false, + "enable_tls": false, + "address": ":1935", + "address_tls": ":1936", + "app": "/", + "token": "" + }, + "srt": { + "enable": false, + "address": ":6000", + "passphrase": "", + "token": "", + "log": { + "enable": false, + "topics": [] + } + }, + "ffmpeg": { + "binary": "ffmpeg", + "max_processes": 0, + "access": { + "input": { + "allow": [], + "block": [] + }, + "output": { + "allow": [], + "block": [] + } + }, + "log": { + "max_lines": 50, + "max_history": 3 + } + }, + "playout": { + "enable": false, + "min_port": 0, + "max_port": 0 + }, + "debug": { + "profiling": false, + "force_gc": 0 + }, + "metrics": { + "enable": false, + "enable_prometheus": false, + "range_sec": 300, + "interval_sec": 2 + }, + "sessions": { + "enable": true, + "ip_ignorelist": [ + "127.0.0.1/32", + "::1/128" + ], + "session_timeout_sec": 30, + "persist": false, + "persist_interval_sec": 300, + "max_bitrate_mbit": 0, + "max_sessions": 0 + }, + "service": { + "enable": false, + "token": "", + "url": "https://service.datarhei.com" + }, + "router": { + "blocked_prefixes": [ + "/api" + ], + "routes": {}, + "ui_path": "" + } +} \ No newline at end of file diff --git a/config/store/json.go b/config/store/json.go index b4456635..b4cd2db9 100644 --- a/config/store/json.go +++ b/config/store/json.go @@ -118,48 +118,12 @@ func (c *jsonStore) load(cfg *config.Config) error { return err } - dataV3 := &config.Data{} - - version := DataVersion{} - - if err = gojson.Unmarshal(jsondata, &version); err != nil { - return json.FormatError(jsondata, err) + data, err := migrate(jsondata) + if err != nil { + return err } - if version.Version == 1 { - dataV1 := &v1.Data{} - - if err = gojson.Unmarshal(jsondata, dataV1); err != nil { - return json.FormatError(jsondata, err) - } - - dataV2, err := v2.UpgradeV1ToV2(dataV1) - if err != nil { - return err - } - - dataV3, err = config.UpgradeV2ToV3(dataV2) - if err != nil { - return err - } - } else if version.Version == 2 { - dataV2 := &v2.Data{} - - if err = gojson.Unmarshal(jsondata, dataV2); err != nil { - return json.FormatError(jsondata, err) - } - - dataV3, err = config.UpgradeV2ToV3(dataV2) - if err != nil { - return err - } - } else if version.Version == 3 { - if err = gojson.Unmarshal(jsondata, dataV3); err != nil { - return json.FormatError(jsondata, err) - } - } - - cfg.Data = *dataV3 + cfg.Data = *data cfg.LoadedAt = time.Now() cfg.UpdatedAt = cfg.LoadedAt @@ -202,3 +166,55 @@ func (c *jsonStore) store(data *config.Config) error { return nil } + +func migrate(jsondata []byte) (*config.Data, error) { + data := &config.Data{} + version := DataVersion{} + + if err := gojson.Unmarshal(jsondata, &version); err != nil { + return nil, json.FormatError(jsondata, err) + } + + if version.Version == 1 { + dataV1 := &v1.New().Data + + if err := gojson.Unmarshal(jsondata, dataV1); err != nil { + return nil, json.FormatError(jsondata, err) + } + + dataV2, err := v2.UpgradeV1ToV2(dataV1) + if err != nil { + return nil, err + } + + dataV3, err := config.UpgradeV2ToV3(dataV2) + if err != nil { + return nil, err + } + + data = dataV3 + } else if version.Version == 2 { + dataV2 := &v2.New().Data + + if err := gojson.Unmarshal(jsondata, dataV2); err != nil { + return nil, json.FormatError(jsondata, err) + } + + dataV3, err := config.UpgradeV2ToV3(dataV2) + if err != nil { + return nil, err + } + + data = dataV3 + } else if version.Version == 3 { + dataV3 := &config.New().Data + + if err := gojson.Unmarshal(jsondata, dataV3); err != nil { + return nil, json.FormatError(jsondata, err) + } + + data = dataV3 + } + + return data, nil +} diff --git a/config/store/json_test.go b/config/store/json_test.go new file mode 100644 index 00000000..6b412c14 --- /dev/null +++ b/config/store/json_test.go @@ -0,0 +1,50 @@ +package store + +import ( + "encoding/json" + "os" + "testing" + "time" + + "github.com/datarhei/core/v16/config" + + "github.com/stretchr/testify/require" +) + +func TestMigrationV1ToV3(t *testing.T) { + jsondatav1, err := os.ReadFile("./fixtures/config_v1.json") + require.NoError(t, err) + + jsondatav3, err := os.ReadFile("./fixtures/config_v1_v3.json") + require.NoError(t, err) + + datav3 := config.New() + json.Unmarshal(jsondatav3, datav3) + + data, err := migrate(jsondatav1) + require.NoError(t, err) + + datav3.Data.CreatedAt = time.Time{} + data.CreatedAt = time.Time{} + + require.Equal(t, datav3.Data, *data) +} + +func TestMigrationV2ToV3(t *testing.T) { + jsondatav2, err := os.ReadFile("./fixtures/config_v2.json") + require.NoError(t, err) + + jsondatav3, err := os.ReadFile("./fixtures/config_v2_v3.json") + require.NoError(t, err) + + datav3 := config.New() + json.Unmarshal(jsondatav3, datav3) + + data, err := migrate(jsondatav2) + require.NoError(t, err) + + datav3.Data.CreatedAt = time.Time{} + data.CreatedAt = time.Time{} + + require.Equal(t, datav3.Data, *data) +} diff --git a/config/value/auth0.go b/config/value/auth0.go index a746fcf7..8d19b4f1 100644 --- a/config/value/auth0.go +++ b/config/value/auth0.go @@ -28,6 +28,7 @@ func NewTenantList(p *[]Auth0Tenant, val []Auth0Tenant, separator string) *Tenan } *p = val + return v } diff --git a/config/value/network.go b/config/value/network.go index 7cf78e49..62a9257d 100644 --- a/config/value/network.go +++ b/config/value/network.go @@ -18,6 +18,7 @@ type Address string func NewAddress(p *string, val string) *Address { *p = val + return (*Address)(p) } @@ -66,7 +67,9 @@ func NewCIDRList(p *[]string, val []string, separator string) *CIDRList { p: p, separator: separator, } + *p = val + return v } @@ -120,7 +123,9 @@ func NewCORSOrigins(p *[]string, val []string, separator string) *CORSOrigins { p: p, separator: separator, } + *p = val + return v } @@ -161,6 +166,7 @@ type Port int func NewPort(p *int, val int) *Port { *p = val + return (*Port)(p) } @@ -197,6 +203,7 @@ type URL string func NewURL(p *string, val string) *URL { *p = val + return (*URL)(p) } @@ -238,6 +245,7 @@ type Email string func NewEmail(p *string, val string) *Email { *p = val + return (*Email)(p) } diff --git a/config/value/os.go b/config/value/os.go index 3139a2ab..dddfdc5e 100644 --- a/config/value/os.go +++ b/config/value/os.go @@ -14,6 +14,7 @@ type MustDir string func NewMustDir(p *string, val string) *MustDir { *p = val + return (*MustDir)(p) } @@ -55,6 +56,7 @@ type Dir string func NewDir(p *string, val string) *Dir { *p = val + return (*Dir)(p) } @@ -96,6 +98,7 @@ type Exec string func NewExec(p *string, val string) *Exec { *p = val + return (*Exec)(p) } @@ -129,6 +132,7 @@ type File string func NewFile(p *string, val string) *File { *p = val + return (*File)(p) } @@ -170,6 +174,7 @@ type AbsolutePath string func NewAbsolutePath(p *string, val string) *AbsolutePath { *p = filepath.Clean(val) + return (*AbsolutePath)(p) } diff --git a/config/value/primitives.go b/config/value/primitives.go index 34057008..701db26b 100644 --- a/config/value/primitives.go +++ b/config/value/primitives.go @@ -11,6 +11,7 @@ type String string func NewString(p *string, val string) *String { *p = val + return (*String)(p) } @@ -43,7 +44,9 @@ func NewStringList(p *[]string, val []string, separator string) *StringList { p: p, separator: separator, } + *p = val + return v } @@ -149,6 +152,7 @@ type Bool bool func NewBool(p *bool, val bool) *Bool { *p = val + return (*Bool)(p) } @@ -179,6 +183,7 @@ type Int int func NewInt(p *int, val int) *Int { *p = val + return (*Int)(p) } @@ -209,6 +214,7 @@ type Int64 int64 func NewInt64(p *int64, val int64) *Int64 { *p = val + return (*Int64)(p) } @@ -239,6 +245,7 @@ type Uint64 uint64 func NewUint64(p *uint64, val uint64) *Uint64 { *p = val + return (*Uint64)(p) } diff --git a/config/value/time.go b/config/value/time.go index c0be6e98..7fe3fa71 100644 --- a/config/value/time.go +++ b/config/value/time.go @@ -8,6 +8,7 @@ type Time time.Time func NewTime(p *time.Time, val time.Time) *Time { *p = val + return (*Time)(p) }