Allow to store HLS on diskfs

This commit is contained in:
Ingo Oppermann 2022-07-18 10:04:59 +02:00
parent 2ccec4873d
commit b5f0fe386e
No known key found for this signature in database
GPG Key ID: 2AB32426E9DD229E
10 changed files with 190 additions and 168 deletions

View File

@ -2,6 +2,7 @@
#### v1.1.0 > v1.2.0
- Add allow writing HLS to disk
- Add audio pan filter
- Add video rotation filter ([#347](https://github.com/datarhei/restreamer/discussions/347))
- Add video h/v flip filter
@ -27,6 +28,7 @@
- Fix VAAPI encoder
Dependency:
- datarhei Core v16.9.0+
#### v1.0.0 > v1.1.0

View File

@ -61,6 +61,19 @@ export default function Control(props) {
</Typography>
</Grid>
*/}
<Grid item xs={12}>
<Select label={<Trans>Storage</Trans>} value={settings.storage} onChange={handleChange('storage')}>
<MenuItem value="memfs">
<Trans>In-Memory (recommended)</Trans>
</MenuItem>
<MenuItem value="diskfs">
<Trans>Disk</Trans>
</MenuItem>
</Select>
<Typography variant="caption">
<Trans>Where to store the HLS playlist and segments.</Trans>
</Typography>
</Grid>
<Grid item xs={12}>
<Select label={<Trans>EXT-X-VERSION</Trans>} value={settings.version} onChange={handleChange('version')}>
<MenuItem value={3}>3</MenuItem>
@ -100,7 +113,11 @@ export default function Control(props) {
</Typography>
</Grid>
<Grid item xs={12}>
<Checkbox label={<Trans>Master playlist (increases browser/client compatibility)</Trans>} checked={settings.master_playlist} onChange={handleChange('master_playlist')} />
<Checkbox
label={<Trans>Master playlist (increases browser/client compatibility)</Trans>}
checked={settings.master_playlist}
onChange={handleChange('master_playlist')}
/>
</Grid>
<Grid item xs={12}>
<Checkbox label={<Trans>Automatic cleanup of all media data</Trans>} checked={settings.cleanup} onChange={handleChange('cleanup')} />

View File

@ -44,37 +44,29 @@ export default function Control(props) {
const items = [];
if (props.sources.includes('hls+memfs')) {
items.push(
<MenuItem key="hls+memfs" value="hls+memfs">
HLS (memfs)
</MenuItem>
);
}
items.push(
<MenuItem key="hls+memfs" value="hls+memfs" disabled={!props.sources.includes('hls+memfs')}>
HLS (memfs)
</MenuItem>
);
if (props.sources.includes('hls+diskfs')) {
items.push(
<MenuItem key="hls+diskfs" value="hls+diskfs">
HLS (diskfs)
</MenuItem>
);
}
items.push(
<MenuItem key="hls+diskfs" value="hls+diskfs" disabled={!props.sources.includes('hls+diskfs')}>
HLS (diskfs)
</MenuItem>
);
if (props.sources.includes('rtmp')) {
items.push(
<MenuItem key="rtmp" value="rtmp">
RTMP
</MenuItem>
);
}
items.push(
<MenuItem key="rtmp" value="rtmp" disabled={!props.sources.includes('rtmp')}>
RTMP
</MenuItem>
);
if (props.sources.includes('srt')) {
items.push(
<MenuItem key="srt" value="srt">
SRT
</MenuItem>
);
}
items.push(
<MenuItem key="srt" value="srt" disabled={!props.sources.includes('srt')}>
SRT
</MenuItem>
);
return (
<Grid container spacing={2}>

View File

@ -936,16 +936,16 @@ class Restreamer {
return await this._getResources();
}
// Get all HTTP addresses
GetHTTPAddresses() {
// Get the public HTTP address
GetPublicHTTPAddress() {
const config = this.ConfigActive();
const address = (config.http.secure === true ? 'https://' : 'http://') + config.http.host;
return [address];
return address;
}
// Get all RTMP/SRT/SNAPSHOT+MEMFS/HLS+MEMFS addresses
GetAddresses(what, channelId) {
GetPublicAddress(what, channelid) {
const config = this.ConfigActive();
const host = config.hostname;
@ -963,7 +963,7 @@ class Restreamer {
}
}
if (what && what === 'rtmp') {
if (what === 'rtmp') {
// rtmp/s
const cfg = config.source.network.rtmp;
const port = getPort(cfg.host);
@ -977,31 +977,56 @@ class Restreamer {
`://${host}${port}` +
(cfg.app.length !== 0 ? cfg.app : '') +
'/' +
channelId +
channelid +
'.stream' +
(cfg.token.length !== 0 ? `?token=${cfg.token}` : '');
} else if (what && what === 'srt') {
} else if (what === 'srt') {
// srt
const cfg = config.source.network.srt;
const port = getPort(cfg.host);
address =
`srt://${host}${port}/?mode=caller&transtype=live&streamid=#!:m=request,r=${channelId}` +
`srt://${host}${port}/?mode=caller&transtype=live&streamid=#!:m=request,r=${channelid}` +
(cfg.token.length !== 0 ? `,token=${cfg.token}` : '') +
(cfg.passphrase.length !== 0 ? `&passphrase=${cfg.passphrase}` : '');
} else if (what && what === 'snapshot+memfs') {
} else if (what === 'snapshot+memfs') {
// snapshot+memfs
const port = getPort(config.source.network.hls.host);
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/memfs/${channelId}.jpg`;
} else {
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/` + this.GetChannelPosterPath(channelid, 'memfs');
} else if (what === 'snapshot+diskfs') {
// snapshot+diskfs
const port = getPort(config.source.network.hls.host);
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/` + this.GetChannelPosterPath(channelid, 'diskfs');
} else if (what === 'hls+memfs') {
// hls+memfs
const port = getPort(config.source.network.hls.host);
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/memfs/${channelId}.m3u8`;
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/` + this.GetChannelManifestPath(channelid, 'memfs');
} else if (what === 'hls+diskfs') {
// hls+diskfs
const port = getPort(config.source.network.hls.host);
address = (config.http.secure === true ? 'https://' : 'http://') + `${host}${port}/` + this.GetChannelManifestPath(channelid, 'diskfs');
} else if (what === 'player') {
// player
address = (config.http.secure === true ? 'https://' : 'http://') + `${config.http.host}/` + this.GetChannelPlayerPath(channelid);
}
return [address];
return address;
}
// Get the iframe codes for the player
GetPublicIframeCode(channelid) {
const channel = this.GetChannel(channelid);
if (channel === null) {
return '';
}
const address = this.GetPublicHTTPAddress();
return `<iframe src="${address}/${channel.channelid}.html" width="640" height="360" frameborder="no" scrolling="no" allowfullscreen="true"></iframe>`;
}
// Channels
@ -1182,7 +1207,7 @@ class Restreamer {
channelid: channel.channelid,
name: channel.name,
available: channel.available,
thumbnail: this.Address() + '/' + this.GetChannelPosterUrl(channel.channelid),
thumbnail: this.GetChannelAddress('snapshot+memfs', channel.channelid),
egresses: Array.from(channel.egresses.keys()),
});
}
@ -1201,7 +1226,7 @@ class Restreamer {
channelid: channel.channelid,
name: channel.name,
available: channel.available,
thumbnail: this.Address() + '/' + this.GetChannelPosterUrl(channel.channelid),
thumbnail: this.GetChannelAddress('snapshot+memfs', channel.channelid),
egresses: Array.from(channel.egresses.keys()),
};
}
@ -1267,16 +1292,46 @@ class Restreamer {
return this.channel.channelid;
}
// Get the URL for the stream
GetChannelManifestUrl(channelid) {
return `memfs/${channelid}.m3u8`;
// Get the path for the HLS manifest
GetChannelManifestPath(channelid, storage) {
if (!storage) {
storage = 'memfs';
}
let url = `${channelid}.m3u8`;
if (storage === 'memfs') {
url = 'memfs/' + url;
}
return url;
}
// Get the URL for the poster image
GetChannelPosterUrl(channelid) {
// Get the path for the poster image
GetChannelPosterPath(channelid, storage) {
return `memfs/${channelid}.jpg`;
}
// Get the path for the player
GetChannelPlayerPath(channelid) {
return `${channelid}.html`;
}
GetChannelAddress(what, channelid) {
const address = this.Address();
if (what === 'hls+memfs') {
return `${address}/${this.GetChannelManifestPath(channelid, 'memfs')}`;
} else if (what === 'hls+diskfs') {
return `${address}/${this.GetChannelManifestPath(channelid, 'diskfs')}`;
} else if (what === 'snapshot+memfs') {
return `${address}/${this.GetChannelPosterPath(channelid, 'memfs')}`;
} else if (what === 'snapshot+diskfs') {
return `${address}/${this.GetChannelPosterPath(channelid, 'diskfs')}`;
} else if (what === 'player') {
return `${address}/${this.GetChannelPlayerPath(channelid)}`;
}
}
// Sessions
async CurrentSessions() {
@ -1421,59 +1476,6 @@ class Restreamer {
return await this.GetDebug(channel.id);
}
GetIngestAddresses(channelid) {
const channel = this.GetChannel(channelid);
if (channel === null) {
return [];
}
const addresses = this.GetHTTPAddresses();
return addresses.map((address) => {
return `${address}/${channel.channelid}.html`;
});
}
// Get the iframe codes for the player
GetIngestIframeCodes(channelid) {
const channel = this.GetChannel(channelid);
if (channel === null) {
return [];
}
const addresses = this.GetHTTPAddresses();
const codes = [];
for (let address of addresses) {
codes.push(
`<iframe src="${address}/${channel.channelid}.html" width="640" height="360" frameborder="no" scrolling="no" allowfullscreen="true"></iframe>`
);
}
return codes;
}
// Get the URL for the HLS manifest
GetIngestManifestUrl(channelid) {
return this.GetChannelManifestUrl(channelid);
}
// Get the URL for poster image
GetIngestPosterUrl(channelid) {
return this.GetChannelPosterUrl(channelid);
}
// Get the URL for poster image
GetIngestPosterUrlAddresses(channelid) {
const poster = this.GetChannelPosterUrl(channelid);
const addresses = this.GetHTTPAddresses();
return addresses.map((address) => {
return `${address}/${poster}`;
});
}
// Start the ingest process
async StartIngest(channelid) {
const channel = this.GetChannel(channelid);
@ -1537,7 +1539,7 @@ class Restreamer {
// Upsert the ingest process
async UpsertIngest(channelid, global, inputs, outputs, control) {
const channel = this.GetChannel(channelid);
if (channel === null) {
if (!channel) {
return [null, { message: 'Unknown channel ID' }];
}
@ -1602,7 +1604,7 @@ class Restreamer {
}
// Injects a metadata link as title
const metadata = `${this.GetHTTPAddresses()[0]}/${channel.channelid}/oembed.json`;
const metadata = `${this.GetPublicHTTPAddress()}/${channel.channelid}/oembed.json`;
const metadata_options = ['-metadata', `title=${metadata}`, '-metadata', 'service_provider=datarhei-Restreamer'];
output.options.push(...metadata_options);
@ -1766,7 +1768,8 @@ class Restreamer {
return [null, { message: 'Unknown channel ID' }];
}
const hlsStore = 'memfs';
// Set hls storage endpoint
const hlsStorage = control.hls.storage;
const snapshot = {
type: 'ffmpeg',
@ -1775,7 +1778,7 @@ class Restreamer {
input: [
{
id: 'input_0',
address: `{${hlsStore}}/${channel.channelid}.m3u8`,
address: `{${hlsStorage}}/${channel.channelid}.m3u8`,
options: [],
},
],
@ -1980,11 +1983,11 @@ class Restreamer {
name: metadata.meta.name,
description: metadata.meta.description,
author_name: metadata.meta.author.name,
author_url: this.GetIngestAddresses(channelid)[0],
author_url: this.GetPublicAddress('player', channelid),
license: metadata.license,
iframecode: this.GetIngestIframeCodes(channelid)[0],
poster: this.GetIngestPosterUrl(channelid),
poster_url: this.GetIngestPosterUrlAddresses(channelid)[0],
iframecode: this.GetPublicIframeCode(channelid),
poster: this.GetChannelPosterPath(channelid, metadata.control.hls.storage),
poster_url: this.GetPublicAddress('snapshot+memfs', channelid),
width: 640,
height: 360,
chromecast: metadata.player.chromecast,
@ -2026,8 +2029,8 @@ class Restreamer {
const playerConfig = {
...metadata.player,
source: this.GetIngestManifestUrl(channelid),
poster: this.GetIngestPosterUrl(channelid),
source: this.GetChannelManifestPath(channelid, metadata.control.hls.storage),
poster: this.GetChannelPosterPath(channelid, metadata.control.hls.storage),
license: {
license: metadata.license,
title: metadata.meta.name,
@ -2181,7 +2184,7 @@ class Restreamer {
channel_creator_description: ingestMetadata.meta.author.description,
channel_creator_description_html: ingestMetadata.meta.author.description.replace(/(?:\r\n|\r|\n)/g, '<br />'),
channel_license: ingestMetadata.license,
channel_poster: this.GetIngestPosterUrl(item.channelid),
channel_poster: this.GetChannelPosterPath(item.channelid, ingestMetadata.control.hls.storage),
channel_width: 640,
channel_height: 360,
};
@ -2830,9 +2833,20 @@ class Restreamer {
return null;
}
const regex = /([a-z]+):\/\/[^/]+(?:\/[0-9A-Za-z-_.~/%:=&?]+)?/gm;
const regex = /(?:([a-z]+):)?\/[^\s]*/gm;
const replace = (s) => {
return s.replaceAll(regex, '$1://[anonymized]');
return s.replaceAll(regex, (match, scheme) => {
if (scheme) {
return `${scheme}://[anonymized]`;
}
const pathElm = match.split('/').filter((p) => p.length !== 0);
if (pathElm.length < 2) {
return match;
}
return `/[anonymized]/${pathElm.pop()}`;
});
};
if (p.config) {

View File

@ -360,19 +360,19 @@ const getSRTAddress = (host, name, token, passphrase) => {
const getHLS = (config, name) => {
const url = getHLSAddress(config.hls.host, config.hls.credentials, config.hls.name, config.hls.secure);
return [url];
return url;
};
const getRTMP = (config) => {
const url = getRTMPAddress(config.rtmp.host, config.rtmp.app, config.rtmp.name, config.rtmp.token, config.rtmp.secure);
return [url];
return url;
};
const getSRT = (config) => {
const url = getSRTAddress(config.srt.host, config.srt.name, config.srt.token, config.srt.passphrase);
return [url];
return url;
};
const getLocalHLS = (config, name) => {
@ -639,7 +639,7 @@ function PushHLS(props) {
const classes = useStyles();
const config = props.config;
const HLSs = getHLS(config);
const HLS = getHLS(config);
return (
<Grid container alignItems="flex-start" spacing={2} className={classes.gridContainer}>
@ -650,7 +650,7 @@ function PushHLS(props) {
</Grid>
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={HLSs.length} value={HLSs.join('\n')} readOnly allowCopy />
<Textarea rows={1} value={HLS} readOnly allowCopy />
</BoxTextarea>
</Grid>
<Grid item xs={12}>
@ -685,7 +685,7 @@ function PushRTMP(props) {
</Grid>
);
} else {
const RTMPs = getRTMP(config);
const RTMP = getRTMP(config);
form = (
<Grid container alignItems="flex-start" spacing={2} className={classes.gridContainer}>
@ -696,7 +696,7 @@ function PushRTMP(props) {
</Grid>
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={RTMPs.length} value={RTMPs.join('\n')} readOnly allowCopy />
<Textarea rows={1} value={RTMP} readOnly allowCopy />
</BoxTextarea>
</Grid>
<Grid item xs={12}>
@ -734,7 +734,7 @@ function PushSRT(props) {
</Grid>
);
} else {
const SRTs = getSRT(config);
const SRT = getSRT(config);
form = (
<Grid container alignItems="flex-start" spacing={2} className={classes.gridContainer}>
@ -745,7 +745,7 @@ function PushSRT(props) {
</Grid>
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={SRTs.length} value={SRTs.join('\n')} readOnly allowCopy />
<Textarea rows={1} value={SRT} readOnly allowCopy />
</BoxTextarea>
</Grid>
<Grid item xs={12}>

View File

@ -39,7 +39,7 @@ function Source(props) {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const HLSs = S.func.getHLS(config, settings.push.name);
const HLS = S.func.getHLS(config, settings.push.name);
return (
<React.Fragment>
@ -48,13 +48,11 @@ function Source(props) {
<Trans>Send stream to this address:</Trans>
</Typography>
</Grid>
{HLSs.length !== 0 && (
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={HLSs.length} value={HLSs.join('\n')} readOnly allowCopy />
</BoxTextarea>
</Grid>
)}
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={1} value={HLS} readOnly allowCopy />
</BoxTextarea>
</Grid>
</React.Fragment>
);
}

View File

@ -59,7 +59,7 @@ function Source(props) {
);
}
const RTMPs = S.func.getRTMP(config, settings.push.name);
const RTMP = S.func.getRTMP(config, settings.push.name);
return (
<React.Fragment>
@ -68,13 +68,11 @@ function Source(props) {
<Trans>Send stream to this address:</Trans>
</Typography>
</Grid>
{RTMPs.length !== 0 && (
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={RTMPs.length} value={RTMPs.join('\n')} readOnly allowCopy />
</BoxTextarea>
</Grid>
)}
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={1} value={RTMP} readOnly allowCopy />
</BoxTextarea>
</Grid>
</React.Fragment>
);
}

View File

@ -59,7 +59,7 @@ function Source(props) {
);
}
const SRTs = S.func.getSRT(config, settings.push.name);
const SRT = S.func.getSRT(config, settings.push.name);
return (
<React.Fragment>
@ -68,13 +68,11 @@ function Source(props) {
<Trans>Send stream to this address:</Trans>
</Typography>
</Grid>
{SRTs.length !== 0 && (
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={SRTs.length} value={SRTs.join('\n')} readOnly allowCopy />
</BoxTextarea>
</Grid>
)}
<Grid item xs={12}>
<BoxTextarea>
<Textarea rows={1} value={SRT} readOnly allowCopy />
</BoxTextarea>
</Grid>
</React.Fragment>
);
}

View File

@ -82,7 +82,6 @@ export default function Main(props) {
const [$config, setConfig] = React.useState(null);
const navigate = useNavigate();
const address = props.restreamer.Address() + '/';
useInterval(async () => {
await update();
@ -282,9 +281,10 @@ export default function Main(props) {
return null;
}
const storage = $metadata.control.hls.storage;
const channel = props.restreamer.GetChannel(_channelid);
const manifest = props.restreamer.GetIngestManifestUrl(_channelid);
const poster = props.restreamer.GetIngestPosterUrl(_channelid);
const manifest = props.restreamer.GetChannelAddress('hls+' + storage, _channelid);
const poster = props.restreamer.GetChannelAddress('snapshot+' + storage, _channelid);
let title = <Trans>Main channel</Trans>;
if (channel && channel.name && channel.name.length !== 0) {
@ -380,7 +380,7 @@ export default function Main(props) {
</Grid>
)}
{$state.state === 'connected' && (
<Player type="videojs-internal" source={address + manifest} poster={address + poster} autoplay mute controls />
<Player type="videojs-internal" source={manifest} poster={poster} autoplay mute controls />
)}
</Grid>
</Grid>
@ -398,7 +398,7 @@ export default function Main(props) {
variant="outlined"
color="default"
size="small"
value={props.restreamer.GetAddresses('hls+memfs', _channelid)}
value={props.restreamer.GetPublicAddress('hls+' + storage, _channelid)}
>
<Trans>HLS</Trans>
</CopyButton>
@ -407,7 +407,7 @@ export default function Main(props) {
variant="outlined"
color="default"
size="small"
value={props.restreamer.GetAddresses('rtmp', _channelid)}
value={props.restreamer.GetPublicAddress('rtmp', _channelid)}
>
<Trans>RTMP</Trans>
</CopyButton>
@ -417,7 +417,7 @@ export default function Main(props) {
variant="outlined"
color="default"
size="small"
value={props.restreamer.GetAddresses('srt', _channelid)}
value={props.restreamer.GetPublicAddress('srt', _channelid)}
>
<Trans>SRT</Trans>
</CopyButton>
@ -426,7 +426,7 @@ export default function Main(props) {
variant="outlined"
color="default"
size="small"
value={props.restreamer.GetAddresses('snapshot+memfs', _channelid)}
value={props.restreamer.GetPublicAddress('snapshot+memfs', _channelid)}
>
<Trans>Snapshot</Trans>
</CopyButton>

View File

@ -64,15 +64,12 @@ export default function Edit(props) {
const { channelid: _channelid } = useParams();
const { i18n } = useLingui();
const address = props.restreamer.Address();
const iframeCodes = props.restreamer.GetIngestIframeCodes(_channelid);
const manifest = props.restreamer.GetIngestManifestUrl(_channelid);
const poster = props.restreamer.GetIngestPosterUrl(_channelid);
const timeout = React.useRef();
const notify = React.useContext(NotifyContext);
const [$player] = React.useState('videojs-public');
const [$ready, setReady] = React.useState(false);
const [$state, setState] = React.useState('disconnected');
const [$data, setData] = React.useState({});
const [$metadata, setMetadata] = React.useState({});
const [$settings, setSettings] = React.useState({});
const [$tab, setTab] = React.useState('embed');
const [$revision, setRevision] = React.useState(0);
@ -104,7 +101,7 @@ export default function Edit(props) {
return;
}
setData(proc.metadata);
setMetadata(proc.metadata);
setState(proc.progress.state);
setSettings(props.restreamer.InitPlayerSettings(proc.metadata.player));
@ -244,12 +241,12 @@ export default function Edit(props) {
const handleDone = async () => {
setSaving(true);
const data = {
...$data,
const metadata = {
...$metadata,
player: $settings,
};
await props.restreamer.SetIngestMetadata(_channelid, data);
await props.restreamer.SetIngestMetadata(_channelid, metadata);
await props.restreamer.UpdatePlayer(_channelid);
setSaving(false);
@ -279,6 +276,12 @@ export default function Edit(props) {
return null;
}
const storage = $metadata.control.hls.storage;
const manifest = props.restreamer.GetChannelAddress('hls+' + storage, _channelid);
const poster = props.restreamer.GetChannelAddress('snapshot+' + storage, _channelid);
const playerAddress = props.restreamer.GetPublicAddress('player');
const iframeCode = props.restreamer.GetPublicIframeCode(_channelid);
return (
<React.Fragment>
<Paper xs={12} md={10}>
@ -299,10 +302,10 @@ export default function Edit(props) {
<Player
key={$revision}
type={$player}
source={address + '/' + manifest}
source={manifest}
autoplay={$settings.autoplay}
mute={$settings.mute}
poster={address + '/' + poster}
poster={poster}
logo={$settings.logo}
colors={$settings.color}
statistics={$settings.statistics}
@ -325,10 +328,10 @@ export default function Edit(props) {
<TabPanel value={$tab} index="embed">
<Grid container spacing={2}>
<Grid item xs={12}>
<TextFieldCopy label={<Trans>Player URL</Trans>} value={address + '/' + _channelid + '.html'} />
<TextFieldCopy label={<Trans>Player URL</Trans>} value={playerAddress} />
</Grid>
<Grid item xs={12}>
<TextFieldCopy label={<Trans>iframe code</Trans>} value={iframeCodes.join('\n')} />
<TextFieldCopy label={<Trans>iframe code</Trans>} value={iframeCode} />
</Grid>
</Grid>
</TabPanel>