From 8e5f5d4cf4c138cab994edf93d7b08849f2439e1 Mon Sep 17 00:00:00 2001 From: Juan Navarro Date: Wed, 5 Jan 2022 15:12:51 +0100 Subject: [PATCH] openvidu-server, browser: Add Simulcast Publisher config (#680) Simulcast is a per-Publisher configuration that allows to enable Simulcast senders on the client's PeerConnection of each sender. Simulcast is a WebRTC feature that sends multiple simultaneous streams with different video qualities, in order to let the media server decide which quality is best for which Subscriber on the receiving side. Enabled by default. --- openvidu-browser/src/OpenVidu/OpenVidu.ts | 5 +++++ openvidu-browser/src/OpenVidu/Session.ts | 1 + openvidu-browser/src/OpenVidu/Stream.ts | 8 +++++++- .../Interfaces/Private/LocalConnectionOptions.ts | 3 ++- .../Public/OpenViduAdvancedConfiguration.ts | 9 +-------- .../Interfaces/Public/PublisherProperties.ts | 12 +++++++++++- .../openvidu/client/internal/ProtocolElements.java | 1 + .../deployments/enterprise/master-node/.env | 10 ++++++++++ .../io/openvidu/server/config/OpenviduConfig.java | 9 +++++++++ .../openvidu/server/core/SessionEventsHandler.java | 13 +++++++++++++ .../openvidu/server/rest/ConfigRestController.java | 1 + .../additional-spring-configuration-metadata.json | 6 ++++++ .../src/main/resources/application.properties | 1 + .../publisher-properties-dialog.component.html | 3 ++- .../openvidu-instance.component.ts | 7 +++---- 15 files changed, 73 insertions(+), 16 deletions(-) diff --git a/openvidu-browser/src/OpenVidu/OpenVidu.ts b/openvidu-browser/src/OpenVidu/OpenVidu.ts index e2de4ff6d..34bdeea19 100644 --- a/openvidu-browser/src/OpenVidu/OpenVidu.ts +++ b/openvidu-browser/src/OpenVidu/OpenVidu.ts @@ -109,6 +109,10 @@ export class OpenVidu { * @hidden */ mediaServer: string; + /** + * @hidden + */ + videoSimulcast: boolean; /** * @hidden */ @@ -257,6 +261,7 @@ export class OpenVidu { publishVideo: (typeof properties.publishVideo !== 'undefined') ? properties.publishVideo : true, resolution: (typeof MediaStreamTrack !== 'undefined' && properties.videoSource instanceof MediaStreamTrack) ? undefined : ((typeof properties.resolution !== 'undefined') ? properties.resolution : '640x480'), videoSource: (typeof properties.videoSource !== 'undefined') ? properties.videoSource : undefined, + videoSimulcast: properties.videoSimulcast, filter: properties.filter }; } else { diff --git a/openvidu-browser/src/OpenVidu/Session.ts b/openvidu-browser/src/OpenVidu/Session.ts index ad2438372..b25c7e103 100644 --- a/openvidu-browser/src/OpenVidu/Session.ts +++ b/openvidu-browser/src/OpenVidu/Session.ts @@ -1540,6 +1540,7 @@ export class Session extends EventDispatcher { this.openvidu.role = opts.role; this.openvidu.finalUserId = opts.finalUserId; this.openvidu.mediaServer = opts.mediaServer; + this.openvidu.videoSimulcast = opts.videoSimulcast; this.capabilities = { subscribe: true, publish: this.openvidu.role !== 'SUBSCRIBER', diff --git a/openvidu-browser/src/OpenVidu/Stream.ts b/openvidu-browser/src/OpenVidu/Stream.ts index 16d8c9b2b..ae5cf7959 100644 --- a/openvidu-browser/src/OpenVidu/Stream.ts +++ b/openvidu-browser/src/OpenVidu/Stream.ts @@ -938,7 +938,8 @@ export class Stream { audio: this.hasAudio, video: this.hasVideo, }, - simulcast: this.session.openvidu.advancedConfiguration.enableSimulcastExperimental || false, + simulcast: + this.outboundStreamOpts.publisherProperties.videoSimulcast ?? this.session.openvidu.videoSimulcast, onIceCandidate: this.connection.sendIceCandidate.bind(this.connection), onIceConnectionStateException: (exceptionName: ExceptionEventName, message: string, data?: any) => { this.session.emitEvent('exception', [new ExceptionEvent(this.session, exceptionName, this, message, data)]) }, iceServers: this.getIceServersConf(), @@ -946,6 +947,11 @@ export class Stream { mediaServer: this.session.openvidu.mediaServer }; + if (this.session.openvidu.mediaServer !== 'mediasoup') { + // Simulcast is only supported by mediasoup + config.simulcast = false; + } + if (reconnect) { this.disposeWebRtcPeer(); } diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts index bc8d2ab3a..8520aad12 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Private/LocalConnectionOptions.ts @@ -33,5 +33,6 @@ export interface LocalConnectionOptions { turnCredential: string; version: string; mediaServer: string; + videoSimulcast: boolean; life: number; -} \ No newline at end of file +} diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts index 2bc8f0464..7327b3c2c 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/OpenViduAdvancedConfiguration.ts @@ -71,11 +71,4 @@ export interface OpenViduAdvancedConfiguration { */ noStreamPlayingEventExceptionTimeout?: number; - /** - * Whether to enable simulcast for Publishers or not. - * - * Default to `false`. - */ - enableSimulcastExperimental?: boolean; - -} \ No newline at end of file +} diff --git a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts index 2ef5df085..dbe0c715e 100644 --- a/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts +++ b/openvidu-browser/src/OpenViduInternal/Interfaces/Public/PublisherProperties.ts @@ -82,6 +82,16 @@ export interface PublisherProperties { */ videoSource?: string | MediaStreamTrack | boolean; + /** + * Send Simulcast video. + * Publishers will encode duplicate video streams with different qualities, + * so the media server is able to select the most appropriate quality stream + * for each Subscriber. + * This setting is honored only if OpenVidu Server was configured to use the + * mediasoup media server. Otherwise, Simulcast will be disabled. + */ + videoSimulcast?: boolean; + /** * **WARNING**: experimental option. This property may change in the near future * @@ -89,4 +99,4 @@ export interface PublisherProperties { */ filter?: Filter; -} \ No newline at end of file +} diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index 4e917016d..85a327d58 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -160,6 +160,7 @@ public class ProtocolElements { public static final String PARTICIPANTJOINED_SESSION_PARAM = "session"; public static final String PARTICIPANTJOINED_VERSION_PARAM = "version"; public static final String PARTICIPANTJOINED_MEDIASERVER_PARAM = "mediaServer"; + public static final String PARTICIPANTJOINED_SIMULCAST_PARAM = "videoSimulcast"; public static final String PARTICIPANTJOINED_RECORD_PARAM = "record"; public static final String PARTICIPANTJOINED_ROLE_PARAM = "role"; public static final String PARTICIPANTJOINED_COTURNIP_PARAM = "coturnIp"; diff --git a/openvidu-server/deployments/enterprise/master-node/.env b/openvidu-server/deployments/enterprise/master-node/.env index ace47e9fd..1db77389c 100644 --- a/openvidu-server/deployments/enterprise/master-node/.env +++ b/openvidu-server/deployments/enterprise/master-node/.env @@ -247,6 +247,16 @@ OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH=1000 # 0 means unconstrained OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 +# Send Simulcast video. +# Publishers will encode duplicate video streams with different qualities, +# so the media server is able to select the most appropriate quality stream +# for each Subscriber. +# This setting is honored only if OpenVidu Server was configured to use the +# mediasoup media server. Otherwise, Simulcast will be disabled. +# Values: true | false +# Default: true +#OPENVIDU_STREAMS_VIDEO_SIMULCAST=true + # All sessions of OpenVidu will try to force this codec. If OPENVIDU_STREAMS_ALLOW_TRANSCODING=true # when a codec can not be forced, transcoding will be allowed # Values: VP8, VP9, H264, NONE diff --git a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java index dede7ea0a..47fc14b13 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java +++ b/openvidu-server/src/main/java/io/openvidu/server/config/OpenviduConfig.java @@ -217,8 +217,13 @@ public class OpenviduConfig { private boolean isTurnadminAvailable = false; // Media Server properties + private MediaServer mediaServerInfo = MediaServer.kurento; + // Media properties + + private boolean streamsVideoSimulcast = false; + // Plain config properties getters public String getCoturnDatabaseDbname() { @@ -281,6 +286,10 @@ public class OpenviduConfig { this.mediaServerInfo = mediaServerInfo; } + public boolean isStreamsVideoSimulcast() { + return this.streamsVideoSimulcast; + } + public String getOpenViduRecordingPath() { return this.openviduRecordingPath; } diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java index 3c54b9398..2ec4fd68b 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/SessionEventsHandler.java @@ -165,6 +165,19 @@ public class SessionEventsHandler { result.addProperty(ProtocolElements.PARTICIPANTJOINED_MEDIASERVER_PARAM, this.openviduConfig.getMediaServer().name()); + switch (this.openviduConfig.getMediaServer()) { + case mediasoup: + // mediasoup supports simulcast + result.addProperty(ProtocolElements.PARTICIPANTJOINED_SIMULCAST_PARAM, + this.openviduConfig.isStreamsVideoSimulcast()); + break; + case kurento: + default: + // Kurento does not support simulcast + result.addProperty(ProtocolElements.PARTICIPANTJOINED_SIMULCAST_PARAM, false); + break; + } + if (participant.getToken() != null) { result.addProperty(ProtocolElements.PARTICIPANTJOINED_RECORD_PARAM, participant.getToken().record()); if (participant.getToken().getRole() != null) { diff --git a/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java b/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java index 9c1d1e2fe..3f3edc397 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java +++ b/openvidu-server/src/main/java/io/openvidu/server/rest/ConfigRestController.java @@ -114,6 +114,7 @@ public class ConfigRestController { json.addProperty("OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH", openviduConfig.getVideoMinRecvBandwidth()); json.addProperty("OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH", openviduConfig.getVideoMaxSendBandwidth()); json.addProperty("OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH", openviduConfig.getVideoMinSendBandwidth()); + json.addProperty("OPENVIDU_STREAMS_VIDEO_SIMULCAST", openviduConfig.isStreamsVideoSimulcast()); json.addProperty("OPENVIDU_STREAMS_FORCED_VIDEO_CODEC", openviduConfig.getOpenviduForcedCodec().name()); json.addProperty("OPENVIDU_STREAMS_ALLOW_TRANSCODING", openviduConfig.isOpenviduAllowingTranscoding()); json.addProperty("OPENVIDU_SESSIONS_GARBAGE_INTERVAL", openviduConfig.getSessionGarbageInterval()); diff --git a/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 6ba031d8c..e34d6a5ed 100644 --- a/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/openvidu-server/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -147,6 +147,12 @@ "description": "Minimum video bandwidth sent from OpenVidu Server to clients, in kbps. 0 means unconstrained", "defaultValue": 300 }, + { + "name": "OPENVIDU_STREAMS_VIDEO_SIMULCAST", + "type": "java.lang.Boolean", + "description": "Send Simulcast video.", + "defaultValue": true + }, { "name": "OPENVIDU_STREAMS_FORCED_VIDEO_CODEC", "type": "java.lang.String", diff --git a/openvidu-server/src/main/resources/application.properties b/openvidu-server/src/main/resources/application.properties index fe3076e7b..6311b88a4 100644 --- a/openvidu-server/src/main/resources/application.properties +++ b/openvidu-server/src/main/resources/application.properties @@ -42,6 +42,7 @@ OPENVIDU_STREAMS_VIDEO_MAX_RECV_BANDWIDTH=1000 OPENVIDU_STREAMS_VIDEO_MIN_RECV_BANDWIDTH=300 OPENVIDU_STREAMS_VIDEO_MAX_SEND_BANDWIDTH=1000 OPENVIDU_STREAMS_VIDEO_MIN_SEND_BANDWIDTH=300 +OPENVIDU_STREAMS_VIDEO_SIMULCAST=true OPENVIDU_STREAMS_FORCED_VIDEO_CODEC=VP8 OPENVIDU_STREAMS_ALLOW_TRANSCODING=false diff --git a/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html b/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html index 816c8b6b2..2044560c6 100644 --- a/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html +++ b/openvidu-testapp/src/app/components/dialogs/publisher-properties-dialog/publisher-properties-dialog.component.html @@ -8,6 +8,7 @@ Publish audio Publish video Mirror + Video Simulcast @@ -52,4 +53,4 @@ - \ No newline at end of file + diff --git a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts index dc46397ec..0c8d8a1e1 100644 --- a/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts +++ b/openvidu-testapp/src/app/components/openvidu-instance/openvidu-instance.component.ts @@ -121,7 +121,8 @@ export class OpenviduInstanceComponent implements OnInit, OnChanges, OnDestroy { resolution: '640x480', mirror: true, publishAudio: true, - publishVideo: true + publishVideo: true, + videoSimulcast: true }; publisherPropertiesAux: PublisherProperties; @@ -230,9 +231,7 @@ export class OpenviduInstanceComponent implements OnInit, OnChanges, OnDestroy { this.OV = new OpenVidu(); - const advancedConfiguration: OpenViduAdvancedConfiguration = { - enableSimulcastExperimental: false - }; + const advancedConfiguration: OpenViduAdvancedConfiguration = {}; if (this.turnConf === 'freeice') { advancedConfiguration.iceServers = 'freeice'; } else if (this.turnConf === 'manual') {