feat: Set degradation preference on video senders (#1229)

* feat: Set degradation preference on video senders
Use 'maintain-framerate' for camera tracks and 'maintain-resolution' for desktop tracks

* ref: apply cached video contraints and degradation preference on unmute
Since the sender encodings are available only after the renegotiation is complete (on chrome/safari), constraints/preferences have to be applied after renegotiation is complete.

modules/RTC/TraceablePeerConnection.js

@@ -6,10 +6,6 @@ import transform from 'sdp-transform';
 
 import * as GlobalOnErrorHandler from '../util/GlobalOnErrorHandler';
 import JitsiRemoteTrack from './JitsiRemoteTrack';
-import {
-    TRACK_ADDED,
-    TRACK_MUTE_CHANGED
-} from '../../JitsiConferenceEvents';
 import * as MediaType from '../../service/RTC/MediaType';
 import * as VideoType from '../../service/RTC/VideoType';
 import LocalSdpMunger from './LocalSdpMunger';
@@ -30,7 +26,8 @@ import * as SignalingEvents from '../../service/RTC/SignalingEvents';
 const logger = getLogger(__filename);
 const MAX_BITRATE = 2500000;
 const DESKSTOP_SHARE_RATE = 500000;
-
+const DEGRADATION_PREFERENCE_CAMERA = 'maintain-framerate';
+const DEGRADATION_PREFERENCE_DESKTOP = 'maintain-resolution';
 /* eslint-disable max-params */
 
 /**
@@ -250,6 +247,11 @@ export default function TraceablePeerConnection(
     this.eventEmitter = rtc.eventEmitter;
     this.rtxModifier = new RtxModifier();
 
+    /**
+     * The height constraint applied on the video sender.
+     */
+    this.senderVideoMaxHeight = null;
+
     // override as desired
     this.trace = (what, info) => {
         logger.debug(what, info);
@@ -340,29 +342,6 @@ export default function TraceablePeerConnection(
         }, 1000);
     }
 
-    // Set sender video constraints when a new local video track is added
-    // to the conference or when it is unmuted.
-    this.senderVideoMaxHeight = null;
-    const maybeSetSenderVideoConstraints = track => {
-        if (track.isLocal()
-            && !track.isMuted()
-            && track.isVideoTrack()
-            && track.videoType === VideoType.CAMERA
-            && this.senderVideoMaxHeight) {
-            this.setSenderVideoConstraint(this.senderVideoMaxHeight)
-                .catch(err => {
-                    logger.error(`Settings sender video constraints failed: ${err}`);
-                });
-        }
-    };
-
-    this.rtc.conference.on(
-        TRACK_ADDED,
-        maybeSetSenderVideoConstraints);
-    this.rtc.conference.on(
-        TRACK_MUTE_CHANGED,
-        maybeSetSenderVideoConstraints);
-
     logger.info(`Create new ${this}`);
 }
 
@@ -1916,6 +1895,43 @@ TraceablePeerConnection.prototype.setAudioTransferActive = function(active) {
     return changed;
 };
 
+/**
+ * Sets the degradation preference on the video sender. This setting determines if
+ * resolution or framerate will be preferred when bandwidth or cpu is constrained.
+ * Sets it to 'maintain-framerate' when a camera track is added to the pc, sets it
+ * to 'maintain-resolution' when a desktop track is being shared instead.
+ * @returns {void}
+ */
+TraceablePeerConnection.prototype.setSenderVideoDegradationPreference = function() {
+    if (!this.peerconnection.getSenders) {
+        logger.debug('Browser does not support RTCRtpSender');
+
+        return;
+    }
+    const localVideoTrack = Array.from(this.localTracks.values()).find(t => t.isVideoTrack());
+    const videoSender = this.findSenderByKind(MediaType.VIDEO);
+
+    if (!videoSender) {
+        return;
+    }
+    const parameters = videoSender.getParameters();
+
+    if (!parameters.encodings || !parameters.encodings.length) {
+        return;
+    }
+    for (const encoding in parameters.encodings) {
+        if (parameters.encodings.hasOwnProperty(encoding)) {
+            const preference = localVideoTrack.videoType === VideoType.CAMERA
+                ? DEGRADATION_PREFERENCE_CAMERA
+                : DEGRADATION_PREFERENCE_DESKTOP;
+
+            logger.info(`Setting video sender degradation preference on ${this} to ${preference}`);
+            parameters.encodings[encoding].degradationPreference = preference;
+        }
+    }
+    videoSender.setParameters(parameters);
+};
+
 /**
  * Sets the max bitrate on the RTCRtpSender so that the
  * bitrate of the enocder doesn't exceed the configured value.
@@ -1925,16 +1941,13 @@ TraceablePeerConnection.prototype.setAudioTransferActive = function(active) {
  * max bitrate is to be configured.
  */
 TraceablePeerConnection.prototype.setMaxBitRate = function(localTrack) {
-    const mediaType = localTrack.type;
     const trackId = localTrack.track.id;
     const videoType = localTrack.videoType;
 
     // No need to set max bitrates on the streams in the following cases.
-    // 1. When an audio track has been replaced.
-    // 2. When a 'camera' track is replaced in plan-b mode, since its a new sender.
-    // 3. When the config.js option for capping the SS bitrate is not enabled.
-    if ((mediaType === MediaType.AUDIO)
-        || (browser.usesPlanB() && !this.options.capScreenshareBitrate)
+    // 1. When a 'camera' track is replaced in plan-b mode, since its a new sender.
+    // 2. When the config.js option for capping the SS bitrate is not enabled.
+    if ((browser.usesPlanB() && !this.options.capScreenshareBitrate)
         || (browser.usesPlanB() && videoType === VideoType.CAMERA)) {
         return;
     }
@@ -2066,8 +2079,13 @@ TraceablePeerConnection.prototype.setRemoteDescription = function(description) {
  * @returns {Promise} promise that will be resolved when the operation is
  * successful and rejected otherwise.
  */
-TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeight) {
-    this.senderVideoMaxHeight = frameHeight;
+TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeight = null) {
+    const newHeight = frameHeight || this.senderVideoMaxHeight;
+
+    this.senderVideoMaxHeight = newHeight;
+    if (!newHeight) {
+        return Promise.resolve();
+    }
     const localVideoTrack = Array.from(this.localTracks.values()).find(t => t.isVideoTrack());
 
     if (!localVideoTrack || localVideoTrack.isMuted() || localVideoTrack.videoType !== VideoType.CAMERA) {
@@ -2093,7 +2111,7 @@ TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeigh
                 // Determine the encodings that need to stay enabled based on the
                 // new frameHeight provided.
                 const encodingsEnabledState = this.tpcUtils.simulcastStreamConstraints
-                    .map(constraint => constraint.height <= frameHeight);
+                    .map(constraint => constraint.height <= newHeight);
                 const videoSender = this.findSenderByKind(MediaType.VIDEO);
 
                 if (!videoSender) {
@@ -2104,7 +2122,7 @@ TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeigh
                 if (!parameters || !parameters.encodings || !parameters.encodings.length) {
                     return Promise.reject(new Error('RTCRtpSendParameters not found for local video track'));
                 }
-                logger.debug(`Setting max height of ${frameHeight} on local video`);
+                logger.debug(`Setting max height of ${newHeight} on local video`);
                 for (const encoding in parameters.encodings) {
                     if (parameters.encodings.hasOwnProperty(encoding)) {
                         parameters.encodings[encoding].active = encodingsEnabledState[encoding];
@@ -2118,13 +2136,13 @@ TraceablePeerConnection.prototype.setSenderVideoConstraint = function(frameHeigh
     // Apply the height constraint on the local camera track
     const aspectRatio = (track.getSettings().width / track.getSettings().height).toPrecision(4);
 
-    logger.debug(`Setting max height of ${frameHeight} on local video`);
+    logger.debug(`Setting max height of ${newHeight} on local video`);
 
     return track.applyConstraints(
         {
             aspectRatio,
             height: {
-                ideal: frameHeight
+                ideal: newHeight
             }
         });
 };

modules/proxyconnection/ProxyConnectionPC.js

@@ -236,19 +236,7 @@ export default class ProxyConnectionPC {
          * A {@code JitsiConference} stub passed to the {@link RTC} module.
          * @type {Object}
          */
-        const conferenceStub = {
-            // FIXME: remove once the temporary code below is gone from
-            //  TraceablePeerConnection.
-            // TraceablePeerConnection:359
-            //  this.rtc.conference.on(
-            //         TRACK_ADDED,
-            //         maybeSetSenderVideoConstraints);
-            //     this.rtc.conference.on(
-            //         TRACK_MUTE_CHANGED,
-            //         maybeSetSenderVideoConstraints);
-            // eslint-disable-next-line no-empty-function
-            on: () => {}
-        };
+        const conferenceStub = {};
 
         /**
          * Create an instance of {@code RTC} as it is required for peer

modules/qualitycontrol/QualityController.js

@@ -43,6 +43,9 @@ export class QualityController {
             });
         this.preferredReceiveMaxFrameHeight
             && mediaSession.setReceiverVideoConstraint(this.preferredReceiveMaxFrameHeight);
+
+        // Set the degradation preference on the local video track.
+        mediaSession.setSenderVideoDegradationPreference();
     }
 
     /**

modules/xmpp/JingleSessionPC.js

@@ -1410,6 +1410,17 @@ export default class JingleSessionPC extends JingleSession {
         return Promise.resolve();
     }
 
+    /**
+     * Sets the degradation preference on the video sender. This setting determines if
+     * resolution or framerate will be preferred when bandwidth or cpu is constrained.
+     * @returns {void}
+     */
+    setSenderVideoDegradationPreference() {
+        if (this._assertNotEnded()) {
+            this.peerconnection.setSenderVideoDegradationPreference();
+        }
+    }
+
     /**
      * @inheritDoc
      */
@@ -1882,15 +1893,18 @@ export default class JingleSessionPC extends JingleSession {
                         finishedCallback /* will be called with en error */);
                     }
 
-                    // Wait for the renegotation to be done if needed (plan-b) before adjusting
-                    // the max bitrates on the video sender.
                     promise.then(() => {
-                        // configure max bitrate only when media is routed
-                        // through JVB. For p2p case, browser takes care of
-                        // adjusting the uplink based on the feedback it
-                        // gets from the peer.
-                        if (newTrack && !this.isP2P) {
-                            this.peerconnection.setMaxBitRate(newTrack);
+                        if (newTrack && newTrack.isVideoTrack()) {
+                            // Set the degradation preference on the new video sender.
+                            this.peerconnection.setSenderVideoDegradationPreference();
+
+                            // Apply the cached video constraints on the new video sender.
+                            this.peerconnection.setSenderVideoConstraint();
+
+                            // Configure max bitrate on the video sender when media is routed through JVB.
+                            if (!this.isP2P) {
+                                this.peerconnection.setMaxBitRate(newTrack);
+                            }
                         }
                         finishedCallback();
                     }, finishedCallback /* will be called with en error */);
@@ -2038,7 +2052,15 @@ export default class JingleSessionPC extends JingleSession {
      */
     addTrackAsUnmute(track) {
         return this._addRemoveTrackAsMuteUnmute(
-            false /* add as unmute */, track);
+            false /* add as unmute */, track)
+            .then(() => {
+                // Apply the video constraints and degradation preference on
+                // the video sender if needed.
+                if (track.isVideoTrack() && browser.doesVideoMuteByStreamRemove()) {
+                    this.peerconnection.setSenderVideoDegradationPreference();
+                    this.peerconnection.setSenderVideoConstraint();
+                }
+            });
     }
 
     /**