From bea10cb1147597a9e4cd8073ac0cd9c26529895d Mon Sep 17 00:00:00 2001 From: Jan Stabenow Date: Tue, 25 Apr 2023 13:56:21 +0200 Subject: [PATCH 1/4] Mod bumps FFmpeg to v5.1.3 --- .github_build/Build.bundle.cuda.env | 2 +- .github_build/Build.bundle.env | 2 +- .github_build/Build.bundle.rpi.env | 2 +- .github_build/Build.bundle.vaapi.env | 2 +- CHANGELOG.md | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github_build/Build.bundle.cuda.env b/.github_build/Build.bundle.cuda.env index 808b4583..be91c889 100644 --- a/.github_build/Build.bundle.cuda.env +++ b/.github_build/Build.bundle.cuda.env @@ -1,3 +1,3 @@ # CORE NVIDIA CUDA BUNDLE -FFMPEG_VERSION=5.1.2 +FFMPEG_VERSION=5.1.3 CUDA_VERSION=11.7.1 diff --git a/.github_build/Build.bundle.env b/.github_build/Build.bundle.env index 060a458e..6758f80d 100644 --- a/.github_build/Build.bundle.env +++ b/.github_build/Build.bundle.env @@ -1,2 +1,2 @@ # CORE BUNDLE -FFMPEG_VERSION=5.1.2 +FFMPEG_VERSION=5.1.3 diff --git a/.github_build/Build.bundle.rpi.env b/.github_build/Build.bundle.rpi.env index 781096cd..3dbe35a9 100644 --- a/.github_build/Build.bundle.rpi.env +++ b/.github_build/Build.bundle.rpi.env @@ -1,2 +1,2 @@ # CORE RASPBERRY-PI BUNDLE -FFMPEG_VERSION=5.1.2 +FFMPEG_VERSION=5.1.3 diff --git a/.github_build/Build.bundle.vaapi.env b/.github_build/Build.bundle.vaapi.env index 060a458e..6758f80d 100644 --- a/.github_build/Build.bundle.vaapi.env +++ b/.github_build/Build.bundle.vaapi.env @@ -1,2 +1,2 @@ # CORE BUNDLE -FFMPEG_VERSION=5.1.2 +FFMPEG_VERSION=5.1.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b82f401..dce53c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add preserve process log history when updating a process - Add support for input framerate data from jsonstats patch - Add number of keyframes and extradata size to process progress data +- Mod bumps FFmpeg to v5.1.3 (datarhei/core:tag bundles) - Fix better naming for storage endpoint documentation - Fix freeing up S3 mounts - Fix URL validation if the path contains FFmpeg specific placeholders From 9b6354ab9441a1db760cb1e916c3e5a9e888a352 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Tue, 25 Apr 2023 15:57:17 +0200 Subject: [PATCH 2/4] Revert commit b58cc8a7ee9fab3fd407fcdf477a52b8a3496062 --- process/process.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/process/process.go b/process/process.go index d1754822..9a8637a2 100644 --- a/process/process.go +++ b/process/process.go @@ -594,9 +594,6 @@ func (p *process) stop(wait bool) error { p.callbacks.onExit = func() { wg.Done() - p.callbacks.lock.Lock() - defer p.callbacks.lock.Unlock() - p.callbacks.onExit = nil } } else { @@ -605,9 +602,6 @@ func (p *process) stop(wait bool) error { cb() wg.Done() - p.callbacks.lock.Lock() - defer p.callbacks.lock.Unlock() - p.callbacks.onExit = cb } } From a2dab2682f079d0401d18ad933feebaa845110a7 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Wed, 26 Apr 2023 09:49:28 +0200 Subject: [PATCH 3/4] Fix not propagating process limits --- ffmpeg/ffmpeg.go | 6 ++++++ process/process.go | 42 +++++++++++++++++++-------------------- restream/restream.go | 13 ++++++++++-- restream/restream_test.go | 23 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/ffmpeg/ffmpeg.go b/ffmpeg/ffmpeg.go index d5f79184..f58d87c1 100644 --- a/ffmpeg/ffmpeg.go +++ b/ffmpeg/ffmpeg.go @@ -32,6 +32,9 @@ type ProcessConfig struct { Reconnect bool ReconnectDelay time.Duration StaleTimeout time.Duration + LimitCPU float64 + LimitMemory uint64 + LimitDuration time.Duration Command []string Parser process.Parser Logger log.Logger @@ -117,6 +120,9 @@ func (f *ffmpeg) New(config ProcessConfig) (process.Process, error) { Reconnect: config.Reconnect, ReconnectDelay: config.ReconnectDelay, StaleTimeout: config.StaleTimeout, + LimitCPU: config.LimitCPU, + LimitMemory: config.LimitMemory, + LimitDuration: config.LimitDuration, Parser: config.Parser, Logger: config.Logger, OnStart: config.OnStart, diff --git a/process/process.go b/process/process.go index 9a8637a2..32b28ae2 100644 --- a/process/process.go +++ b/process/process.go @@ -63,26 +63,19 @@ type Config struct { // Status represents the current status of a process type Status struct { - // State is the current state of the process. See stateType for the known states. - State string - - // States is the cumulative history of states the process had. - States States - - // Order is the wanted condition of process, either "start" or "stop" - Order string - - // Duration is the time since the last change of the state - Duration time.Duration - - // Time is the time of the last change of the state - Time time.Time - - // Used CPU in percent - CPU float64 - - // Used memory in bytes - Memory uint64 + State string // State is the current state of the process. See stateType for the known states. + States States // States is the cumulative history of states the process had. + Order string // Order is the wanted condition of process, either "start" or "stop" + Duration time.Duration // Duration is the time since the last change of the state + Time time.Time // Time is the time of the last change of the state + CPU struct { + Current float64 // Used CPU in percent + Limit float64 // Limit in percent + } + Memory struct { + Current uint64 // Used memory in bytes + Limit uint64 // Limit in bytes + } } // States @@ -390,6 +383,7 @@ func (p *process) getStateString() string { // Status returns the current status of the process func (p *process) Status() Status { cpu, memory := p.limits.Current() + cpuLimit, memoryLimit := p.limits.Limits() p.state.lock.Lock() stateTime := p.state.time @@ -407,10 +401,14 @@ func (p *process) Status() Status { Order: order, Duration: time.Since(stateTime), Time: stateTime, - CPU: cpu, - Memory: memory, } + s.CPU.Current = cpu + s.CPU.Limit = cpuLimit + + s.Memory.Current = memory + s.Memory.Limit = memoryLimit + return s } diff --git a/restream/restream.go b/restream/restream.go index bd5e1482..eb014297 100644 --- a/restream/restream.go +++ b/restream/restream.go @@ -355,6 +355,9 @@ func (r *restream) load() error { Reconnect: t.config.Reconnect, ReconnectDelay: time.Duration(t.config.ReconnectDelay) * time.Second, StaleTimeout: time.Duration(t.config.StaleTimeout) * time.Second, + LimitCPU: t.config.LimitCPU, + LimitMemory: t.config.LimitMemory, + LimitDuration: time.Duration(t.config.LimitWaitFor) * time.Second, Command: t.command, Parser: t.parser, Logger: t.logger, @@ -494,6 +497,9 @@ func (r *restream) createTask(config *app.Config) (*task, error) { Reconnect: t.config.Reconnect, ReconnectDelay: time.Duration(t.config.ReconnectDelay) * time.Second, StaleTimeout: time.Duration(t.config.StaleTimeout) * time.Second, + LimitCPU: t.config.LimitCPU, + LimitMemory: t.config.LimitMemory, + LimitDuration: time.Duration(t.config.LimitWaitFor) * time.Second, Command: t.command, Parser: t.parser, Logger: t.logger, @@ -1179,6 +1185,9 @@ func (r *restream) reloadProcess(id string) error { Reconnect: t.config.Reconnect, ReconnectDelay: time.Duration(t.config.ReconnectDelay) * time.Second, StaleTimeout: time.Duration(t.config.StaleTimeout) * time.Second, + LimitCPU: t.config.LimitCPU, + LimitMemory: t.config.LimitMemory, + LimitDuration: time.Duration(t.config.LimitWaitFor) * time.Second, Command: t.command, Parser: t.parser, Logger: t.logger, @@ -1218,8 +1227,8 @@ func (r *restream) GetProcessState(id string) (*app.State, error) { state.State = status.State state.States.Marshal(status.States) state.Time = status.Time.Unix() - state.Memory = status.Memory - state.CPU = status.CPU + state.Memory = status.Memory.Current + state.CPU = status.CPU.Current state.Duration = status.Duration.Round(10 * time.Millisecond).Seconds() state.Reconnect = -1 state.Command = make([]string, len(task.command)) diff --git a/restream/restream_test.go b/restream/restream_test.go index c142c7fd..64a536a5 100644 --- a/restream/restream_test.go +++ b/restream/restream_test.go @@ -883,3 +883,26 @@ func TestReplacer(t *testing.T) { require.Equal(t, process, rs.tasks["314159265359"].config) } + +func TestProcessLimit(t *testing.T) { + rsi, err := getDummyRestreamer(nil, nil, nil, nil) + require.NoError(t, err) + + process := getDummyProcess() + process.LimitCPU = 61 + process.LimitMemory = 42 + process.Autostart = false + + err = rsi.AddProcess(process) + require.NoError(t, err) + + rs := rsi.(*restream) + + task, ok := rs.tasks[process.ID] + require.True(t, ok) + + status := task.ffmpeg.Status() + + require.Equal(t, float64(61), status.CPU.Limit) + require.Equal(t, uint64(42), status.Memory.Limit) +} From e45f80ed42833c328bb34272a45d43a162bc23a5 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Wed, 26 Apr 2023 09:50:09 +0200 Subject: [PATCH 4/4] Fix tests --- net/url/url_test.go | 2 +- restream/fs/fs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/net/url/url_test.go b/net/url/url_test.go index dff373b8..1ad33409 100644 --- a/net/url/url_test.go +++ b/net/url/url_test.go @@ -55,7 +55,7 @@ func TestScheme(t *testing.T) { require.False(t, r) } -func TestPars(t *testing.T) { +func TestParse(t *testing.T) { u, err := Parse("http://localhost/foobar") require.NoError(t, err) require.Equal(t, &URL{ diff --git a/restream/fs/fs_test.go b/restream/fs/fs_test.go index 0162be1d..2b208096 100644 --- a/restream/fs/fs_test.go +++ b/restream/fs/fs_test.go @@ -78,7 +78,7 @@ func TestMaxAge(t *testing.T) { require.Eventually(t, func() bool { return cleanfs.Files() == 0 - }, 5*time.Second, time.Second) + }, 10*time.Second, time.Second) cleanfs.WriteFileReader("/chunk_3.ts", strings.NewReader("chunk_3")) @@ -96,7 +96,7 @@ func TestMaxAge(t *testing.T) { require.ElementsMatch(t, []string{"/chunk_3.ts"}, names) return true - }, 2*time.Second, time.Second) + }, 5*time.Second, time.Second) cleanfs.Stop() }