feat(multi-stream-support) Configure the senders based on SenderVideoConstraintsV2.

The encodings for local video tracks are confiugured based on the SenderVideoConstraintsV2 constraints received from the bridge. The constraints are sourceName based. Also ignore enableLayerSuspension flag since it has been deprecated and layer suspension has been enabled by default. Also having it off makes screensharing on Chrome in unified plan not work as expected.

JitsiConference.js

@@ -882,6 +882,14 @@ JitsiConference.prototype.getLocalVideoTrack = function() {
     return this.rtc ? this.rtc.getLocalVideoTrack() : null;
 };
 
+/**
+ * Returns all the local video tracks.
+ * @returns {Array<JitsiLocalTrack>}
+ */
+JitsiConference.prototype.getLocalVideoTracks = function() {
+    return this.rtc ? this.rtc.getLocalVideoTracks() : null;
+};
+
 /**
  * Obtains the performance statistics.
  * @returns {Object|null}

modules/RTC/BridgeChannel.js

@@ -2,6 +2,7 @@ import { getLogger } from '@jitsi/logger';
 
 import RTCEvents from '../../service/RTC/RTCEvents';
 import { createBridgeChannelClosedEvent } from '../../service/statistics/AnalyticsEvents';
+import FeatureFlags from '../flags/FeatureFlags';
 import Statistics from '../statistics/statistics';
 import GlobalOnErrorHandler from '../util/GlobalOnErrorHandler';
 
@@ -375,6 +376,25 @@ export default class BridgeChannel {
                 }
                 break;
             }
+            case 'SenderVideoConstraintsV2': {
+                if (FeatureFlags.isSourceNameSignalingEnabled()) {
+                    const { sourceName, idealHeight } = obj;
+
+                    if (typeof sourceName === 'string' && typeof idealHeight === 'number') {
+                        // eslint-disable-next-line object-property-newline
+                        logger.info(`SenderVideoConstraintsV2: ${JSON.stringify({ sourceName, idealHeight })}`);
+                        emitter.emit(
+                            RTCEvents.SENDER_VIDEO_CONSTRAINTS_CHANGED, {
+                                sourceName,
+                                idealHeight
+                            }
+                        );
+                    } else {
+                        logger.error(`Invalid SenderVideoConstraintsV2: ${JSON.stringify(obj)}`);
+                    }
+                }
+                break;
+            }
             case 'ServerHello': {
                 logger.info(`Received ServerHello, version=${obj.version}.`);
                 break;

modules/RTC/RTC.js

@@ -561,6 +561,14 @@ export default class RTC extends Listenable {
         return localVideo.length ? localVideo[0] : undefined;
     }
 
+    /**
+     * Returns all the local video tracks.
+     * @returns {Array<JitsiLocalTrack>}
+     */
+    getLocalVideoTracks() {
+        return this.getLocalTracks(MediaType.VIDEO);
+    }
+
     /**
      * Get local audio track.
      * @returns {JitsiLocalTrack|undefined}

modules/RTC/TraceablePeerConnection.js

@@ -297,6 +297,12 @@ export default function TraceablePeerConnection(
      */
     this._senderVideoMaxHeight = 2160;
 
+    /**
+     * The height constraints to be applied on the sender per local video source (source name as the key).
+     * @type {Map<string, number>}
+     */
+    this._senderMaxHeights = new Map();
+
     // override as desired
     this.trace = (what, info) => {
         logger.debug(what, info);
@@ -2249,10 +2255,32 @@ TraceablePeerConnection.prototype._initializeDtlsTransport = function() {
 /**
  * Configures the stream encodings depending on the video type and the bitrates configured.
  *
+ * @param {JitsiLocalTrack} - The local track for which the sender encodings have to configured.
  * @returns {Promise} promise that will be resolved when the operation is successful and rejected otherwise.
  */
-TraceablePeerConnection.prototype.configureSenderVideoEncodings = function() {
-    return this.setSenderVideoConstraints(this._senderVideoMaxHeight);
+TraceablePeerConnection.prototype.configureSenderVideoEncodings = function(localVideoTrack = null) {
+    if (FeatureFlags.isSourceNameSignalingEnabled()) {
+        if (localVideoTrack) {
+            return this.setSenderVideoConstraints(
+                this._senderMaxHeights.get(localVideoTrack.getSourceName()),
+                localVideoTrack);
+        }
+        const promises = [];
+
+        for (const track of this.getLocalVideoTracks()) {
+            promises.push(this.setSenderVideoConstraints(this._senderMaxHeights.get(track.getSourceName()), track));
+        }
+
+        return Promise.allSettled(promises);
+    }
+
+    let localTrack = localVideoTrack;
+
+    if (!localTrack) {
+        localTrack = this.getLocalVideoTracks()[0];
+    }
+
+    return this.setSenderVideoConstraints(this._senderVideoMaxHeight, localTrack);
 };
 
 TraceablePeerConnection.prototype.setLocalDescription = function(description) {
@@ -2395,9 +2423,10 @@ TraceablePeerConnection.prototype.setRemoteDescription = function(description) {
  * bitrates on the send stream.
  *
  * @param {number} frameHeight - The max frame height to be imposed on the outgoing video stream.
+ * @param {JitsiLocalTrack} - The local track for which the sender constraints have to be applied.
  * @returns {Promise} promise that will be resolved when the operation is successful and rejected otherwise.
  */
-TraceablePeerConnection.prototype.setSenderVideoConstraints = function(frameHeight) {
+TraceablePeerConnection.prototype.setSenderVideoConstraints = function(frameHeight, localVideoTrack) {
     if (frameHeight < 0) {
         throw new Error(`Invalid frameHeight: ${frameHeight}`);
     }
@@ -2407,13 +2436,16 @@ TraceablePeerConnection.prototype.setSenderVideoConstraints = function(frameHeig
         return Promise.resolve();
     }
 
-    this._senderVideoMaxHeight = frameHeight;
-    const localVideoTrack = this.getLocalVideoTracks()[0];
+    if (FeatureFlags.isSourceNameSignalingEnabled()) {
+        this._senderMaxHeights.set(localVideoTrack.getSourceName(), frameHeight);
+    } else {
+        this._senderVideoMaxHeight = frameHeight;
+    }
 
     if (!localVideoTrack || localVideoTrack.isMuted()) {
         return Promise.resolve();
     }
-    const videoSender = this.findSenderByKind(MediaType.VIDEO);
+    const videoSender = this.findSenderForTrack(localVideoTrack.getTrack());
 
     if (!videoSender) {
         return Promise.resolve();

modules/qualitycontrol/SendVideoController.js

@@ -1,5 +1,6 @@
 import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
 import RTCEvents from '../../service/RTC/RTCEvents';
+import FeatureFlags from '../flags/FeatureFlags';
 import MediaSessionEvents from '../xmpp/MediaSessionEvents';
 
 /**
@@ -18,24 +19,24 @@ export default class SendVideoController {
      * @param {RTC} rtc - the rtc instance that is responsible for sending the messages on the bridge channel.
      */
     constructor(conference, rtc) {
-        this.conference = conference;
-        this.layerSuspensionEnabled = conference.options?.config?.enableLayerSuspension ?? true;
-        this.rtc = rtc;
-        this.conference.on(
+        this._conference = conference;
+        this._rtc = rtc;
+
+        /**
+         * Source name based sender constraints.
+         * @type {Map<string, number>};
+         */
+
+        this._sourceSenderConstraints = new Map();
+        this._conference.on(
             JitsiConferenceEvents._MEDIA_SESSION_STARTED,
             session => this._onMediaSessionStarted(session));
-        this.conference.on(
+        this._conference.on(
             JitsiConferenceEvents._MEDIA_SESSION_ACTIVE_CHANGED,
             () => this._propagateSendMaxFrameHeight());
-        this.rtc.on(
+        this._rtc.on(
             RTCEvents.SENDER_VIDEO_CONSTRAINTS_CHANGED,
-            videoConstraints => {
-                // Propagate the sender constraint only if it has changed.
-                if (this._senderVideoConstraints?.idealHeight !== videoConstraints.idealHeight) {
-                    this._senderVideoConstraints = videoConstraints;
-                    this._propagateSendMaxFrameHeight();
-                }
-            });
+            videoConstraints => this._onSenderConstraintsReceived(videoConstraints));
     }
 
     /**
@@ -50,26 +51,52 @@ export default class SendVideoController {
         mediaSession.addListener(
             MediaSessionEvents.REMOTE_VIDEO_CONSTRAINTS_CHANGED,
             session => {
-                if (session === this.conference.getActiveMediaSession()) {
+                if (session === this._conference.getActiveMediaSession()) {
                     this._propagateSendMaxFrameHeight();
                 }
             });
     }
 
+    /**
+     * Propagates the video constraints if they have changed.
+     *
+     * @param {Object} videoConstraints - The sender video constraints received from the bridge.
+     */
+    _onSenderConstraintsReceived(videoConstraints) {
+        if (FeatureFlags.isSourceNameSignalingEnabled()) {
+            const { idealHeight, sourceName } = videoConstraints;
+            const localVideoTracks = this._conference.getLocalVideoTracks() ?? [];
+
+            for (const track of localVideoTracks) {
+                // Propagate the sender constraint only if it has changed.
+                if (track.getSourceName() === sourceName
+                    && (!this._sourceSenderConstraints.has(sourceName)
+                    || this._sourceSenderConstraints.get(sourceName) !== idealHeight)) {
+                    this._sourceSenderConstraints.set(sourceName, idealHeight);
+                    this._propagateSendMaxFrameHeight(sourceName);
+                }
+            }
+        } else if (this._senderVideoConstraints?.idealHeight !== videoConstraints.idealHeight) {
+            this._senderVideoConstraints = videoConstraints;
+            this._propagateSendMaxFrameHeight();
+        }
+    }
+
     /**
      * Figures out the send video constraint as specified by {@link selectSendMaxFrameHeight} and sets it on all media
      * sessions for the reasons mentioned in this class description.
      *
+     * @param {string} sourceName - The source for which sender constraints have changed.
      * @returns {Promise<void[]>}
      * @private
      */
-    _propagateSendMaxFrameHeight() {
-        const sendMaxFrameHeight = this.selectSendMaxFrameHeight();
+    _propagateSendMaxFrameHeight(sourceName = null) {
+        const sendMaxFrameHeight = this.selectSendMaxFrameHeight(sourceName);
         const promises = [];
 
         if (sendMaxFrameHeight >= 0) {
-            for (const session of this.conference.getMediaSessions()) {
-                promises.push(session.setSenderVideoConstraint(sendMaxFrameHeight));
+            for (const session of this._conference.getMediaSessions()) {
+                promises.push(session.setSenderVideoConstraint(sendMaxFrameHeight, sourceName));
             }
         }
 
@@ -80,23 +107,24 @@ export default class SendVideoController {
      * Selects the lowest common value for the local video send constraint by looking at local user's preference and
      * the active media session's receive preference set by the remote party.
      *
+     * @param {string} sourceName - The source for which sender constraints have changed.
      * @returns {number|undefined}
      */
-    selectSendMaxFrameHeight() {
-        const activeMediaSession = this.conference.getActiveMediaSession();
+    selectSendMaxFrameHeight(sourceName = null) {
+        const activeMediaSession = this._conference.getActiveMediaSession();
         const remoteRecvMaxFrameHeight = activeMediaSession
             ? activeMediaSession.isP2P
                 ? activeMediaSession.getRemoteRecvMaxFrameHeight()
-                : this.layerSuspensionEnabled ? this._senderVideoConstraints?.idealHeight : undefined
+                : sourceName ? this._sourceSenderConstraints.get(sourceName) : this._senderVideoConstraints?.idealHeight
             : undefined;
 
-        if (this.preferredSendMaxFrameHeight >= 0 && remoteRecvMaxFrameHeight >= 0) {
-            return Math.min(this.preferredSendMaxFrameHeight, remoteRecvMaxFrameHeight);
+        if (this._preferredSendMaxFrameHeight >= 0 && remoteRecvMaxFrameHeight >= 0) {
+            return Math.min(this._preferredSendMaxFrameHeight, remoteRecvMaxFrameHeight);
         } else if (remoteRecvMaxFrameHeight >= 0) {
             return remoteRecvMaxFrameHeight;
         }
 
-        return this.preferredSendMaxFrameHeight;
+        return this._preferredSendMaxFrameHeight;
     }
 
     /**
@@ -106,7 +134,7 @@ export default class SendVideoController {
      * @returns {Promise<void[]>} - resolved when the operation is complete.
      */
     setPreferredSendMaxFrameHeight(maxFrameHeight) {
-        this.preferredSendMaxFrameHeight = maxFrameHeight;
+        this._preferredSendMaxFrameHeight = maxFrameHeight;
 
         return this._propagateSendMaxFrameHeight();
     }

modules/qualitycontrol/SendVideoController.spec.js

@@ -1,5 +1,6 @@
 import * as JitsiConferenceEvents from '../../JitsiConferenceEvents';
 import RTCEvents from '../../service/RTC/RTCEvents';
+import FeatureFlags from '../flags/FeatureFlags';
 import Listenable from '../util/Listenable';
 import MediaSessionEvents from '../xmpp/MediaSessionEvents';
 
@@ -78,6 +79,10 @@ class MockConference extends Listenable {
         return this.activeMediaSession;
     }
 
+    getLocalVideoTracks() {
+        return [];
+    }
+
     getMediaSessions() {
         return this.mediaSessions;
     }
@@ -108,6 +113,7 @@ describe('SendVideoController', () => {
     beforeEach(() => {
         conference = new MockConference();
         rtc = new MockRTC();
+        FeatureFlags.init({ sourceNameSignaling: false });
         sendVideoController = new SendVideoController(conference, rtc);
         jvbConnection = new MockJingleSessionPC(rtc, false /* isP2P */);
         p2pConnection = new MockJingleSessionPC(rtc, true /* isP2P */);

modules/xmpp/JingleSessionPC.js

@@ -1506,12 +1506,13 @@ export default class JingleSessionPC extends JingleSession {
     /**
      * Sets the resolution constraint on the local camera track.
      * @param {number} maxFrameHeight - The user preferred max frame height.
+     * @param {string} sourceName - The source name of the track.
      * @returns {Promise} promise that will be resolved when the operation is
      * successful and rejected otherwise.
      */
-    setSenderVideoConstraint(maxFrameHeight) {
+    setSenderVideoConstraint(maxFrameHeight, sourceName = null) {
         if (this._assertNotEnded()) {
-            logger.info(`${this} setSenderVideoConstraint: ${maxFrameHeight}`);
+            logger.info(`${this} setSenderVideoConstraint: ${maxFrameHeight}, sourceName: ${sourceName}`);
 
             // RN doesn't support RTCRtpSenders yet, aggresive layer suspension on RN is implemented
             // by changing the media direction in the SDP. This is applicable to jvb sessions only.
@@ -1521,11 +1522,11 @@ export default class JingleSessionPC extends JingleSession {
                 return this.setMediaTransferActive(true, videoActive);
             }
 
-            const promise = typeof maxFrameHeight === 'undefined'
-                ? this.peerconnection.configureSenderVideoEncodings()
-                : this.peerconnection.setSenderVideoConstraints(maxFrameHeight);
+            const jitsiLocalTrack = sourceName
+                ? this.rtc.getLocalVideoTracks().find(track => track.getSourceName() === sourceName)
+                : this.rtc.getLocalVideoTrack();
 
-            return promise;
+            return this.peerconnection.setSenderVideoConstraints(maxFrameHeight, jitsiLocalTrack);
         }
 
         return Promise.resolve();
@@ -2151,7 +2152,7 @@ export default class JingleSessionPC extends JingleSession {
                             logger.debug(`${this} replaceTrack worker: configuring video stream`);
 
                             // Configure the video encodings after the track is replaced.
-                            return this.peerconnection.configureSenderVideoEncodings();
+                            return this.peerconnection.configureSenderVideoEncodings(newTrack);
                         }
                     });
                 })
@@ -2299,7 +2300,7 @@ export default class JingleSessionPC extends JingleSession {
                 // Configure the video encodings after the track is unmuted. If the user joins the call muted and
                 // unmutes it the first time, all the parameters need to be configured.
                 if (track.isVideoTrack()) {
-                    return this.peerconnection.configureSenderVideoEncodings();
+                    return this.peerconnection.configureSenderVideoEncodings(track);
                 }
             });
     }

types/auto/JitsiConference.d.ts

@@ -316,6 +316,11 @@ declare class JitsiConference {
      * @return {JitsiLocalTrack|null}
      */
     getLocalVideoTrack(): any | null;
+    /**
+     * Returns all the local video tracks.
+     * @returns {Array<JitsiLocalTrack>}
+     */
+    getLocalVideoTracks(): Array<any>;
     /**
      * Obtains the performance statistics.
      * @returns {Object|null}

types/auto/modules/RTC/RTC.d.ts

@@ -345,6 +345,11 @@ export default class RTC extends Listenable {
      * @returns {JitsiLocalTrack|undefined}
      */
     getLocalVideoTrack(): JitsiLocalTrack | undefined;
+    /**
+     * Returns all the local video tracks.
+     * @returns {Array<JitsiLocalTrack>}
+     */
+    getLocalVideoTracks(): Array<JitsiLocalTrack>;
     /**
      * Get local audio track.
      * @returns {JitsiLocalTrack|undefined}

types/auto/modules/RTC/TraceablePeerConnection.d.ts

@@ -254,6 +254,11 @@ export default class TraceablePeerConnection {
      * explicitly disabled.
      */
     _senderVideoMaxHeight: number;
+    /**
+     * The height constraints to be applied on the sender per local video source (source name as the key).
+     * @type {Map<string, number>}
+     */
+    _senderMaxHeights: Map<string, number>;
     trace: (what: any, info: any) => void;
     onicecandidate: any;
     onTrack: (evt: any) => void;
@@ -628,9 +633,10 @@ export default class TraceablePeerConnection {
     /**
      * Configures the stream encodings depending on the video type and the bitrates configured.
      *
+     * @param {JitsiLocalTrack} - The local track for which the sender encodings have to configured.
      * @returns {Promise} promise that will be resolved when the operation is successful and rejected otherwise.
      */
-    configureSenderVideoEncodings(): Promise<any>;
+    configureSenderVideoEncodings(localVideoTrack?: any): Promise<any>;
     setLocalDescription(description: any): Promise<any>;
     /**
      * Enables/disables audio media transmission on this peer connection. When
@@ -653,9 +659,10 @@ export default class TraceablePeerConnection {
      * bitrates on the send stream.
      *
      * @param {number} frameHeight - The max frame height to be imposed on the outgoing video stream.
+     * @param {JitsiLocalTrack} - The local track for which the sender constraints have to be applied.
      * @returns {Promise} promise that will be resolved when the operation is successful and rejected otherwise.
      */
-    setSenderVideoConstraints(frameHeight: number): Promise<any>;
+    setSenderVideoConstraints(frameHeight: number, localVideoTrack: any): Promise<any>;
     encodingsEnabledState: boolean[];
     /**
      * Enables/disables video media transmission on this peer connection. When

types/auto/modules/qualitycontrol/SendVideoController.d.ts

@@ -14,10 +14,13 @@ export default class SendVideoController {
      * @param {RTC} rtc - the rtc instance that is responsible for sending the messages on the bridge channel.
      */
     constructor(conference: any, rtc: any);
-    conference: any;
-    layerSuspensionEnabled: any;
-    rtc: any;
-    _senderVideoConstraints: any;
+    _conference: any;
+    _rtc: any;
+    /**
+     * Source name based sender constraints.
+     * @type {Map<string, number>};
+     */
+    _sourceSenderConstraints: Map<string, number>;
     /**
      * Handles the {@link JitsiConferenceEvents.MEDIA_SESSION_STARTED}, that is when the conference creates new media
      * session. It doesn't mean it's already active though. For example the JVB connection may be created after
@@ -27,10 +30,18 @@ export default class SendVideoController {
      * @private
      */
     private _onMediaSessionStarted;
+    /**
+     * Propagates the video constraints if they have changed.
+     *
+     * @param {Object} videoConstraints - The sender video constraints received from the bridge.
+     */
+    _onSenderConstraintsReceived(videoConstraints: any): void;
+    _senderVideoConstraints: any;
     /**
      * Figures out the send video constraint as specified by {@link selectSendMaxFrameHeight} and sets it on all media
      * sessions for the reasons mentioned in this class description.
      *
+     * @param {string} sourceName - The source for which sender constraints have changed.
      * @returns {Promise<void[]>}
      * @private
      */
@@ -39,9 +50,10 @@ export default class SendVideoController {
      * Selects the lowest common value for the local video send constraint by looking at local user's preference and
      * the active media session's receive preference set by the remote party.
      *
+     * @param {string} sourceName - The source for which sender constraints have changed.
      * @returns {number|undefined}
      */
-    selectSendMaxFrameHeight(): number | undefined;
+    selectSendMaxFrameHeight(sourceName?: string): number | undefined;
     /**
      * Sets local preference for max send video frame height.
      *
@@ -49,5 +61,5 @@ export default class SendVideoController {
      * @returns {Promise<void[]>} - resolved when the operation is complete.
      */
     setPreferredSendMaxFrameHeight(maxFrameHeight: number): Promise<void[]>;
-    preferredSendMaxFrameHeight: number;
+    _preferredSendMaxFrameHeight: number;
 }

types/auto/modules/xmpp/JingleSessionPC.d.ts

@@ -355,10 +355,11 @@ export default class JingleSessionPC extends JingleSession {
     /**
      * Sets the resolution constraint on the local camera track.
      * @param {number} maxFrameHeight - The user preferred max frame height.
+     * @param {string} sourceName - The source name of the track.
      * @returns {Promise} promise that will be resolved when the operation is
      * successful and rejected otherwise.
      */
-    setSenderVideoConstraint(maxFrameHeight: number): Promise<any>;
+    setSenderVideoConstraint(maxFrameHeight: number, sourceName?: string): Promise<any>;
     /**
      *
      * @param reasonCondition